Merge pull request #121 from tomato42/better-ca-handling

Better CA certificate handling

cipherscan

@@ -564,7 +564,8 @@ test_cipher_on_target() {
             # signed ones)
             local saved="False"
             if ${OPENSSLBIN} verify "${trust_source[@]}" \
-                -untrusted <(printf "%s" "${current_raw_certificates[@]}") <(echo "$cert") 2>/dev/null | \
+                -untrusted <(printf "%s" "${current_raw_certificates[@]}") \
+                <(echo "$cert") 2>/dev/null | \
                 grep ': OK$' >/dev/null; then
 
                 # if the certificate is an intermediate CA it may be useful
@@ -661,7 +662,8 @@ test_cipher_on_target() {
 # Calculate the average handshake time for a specific ciphersuite
 bench_cipher() {
     local ciphersuite="$1"
-    local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+    local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client $SCLIENTARGS"
+    sslcommand+=" -connect $TARGET -cipher $ciphersuite"
     local t="$(date +%s%N)"
     verbose "Benchmarking handshake on '$TARGET' with ciphersuite '$ciphersuite'"
     for i in $(seq 1 $BENCHMARKITER); do
@@ -690,7 +692,8 @@ get_cipher_pref() {
     elif [[ -e $CACERTS ]]; then
         sslcommand+=" -CAfile $CACERTS"
     fi
-    sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+    sslcommand+=" -trusted_first -status $SCLIENTARGS -connect $TARGET"
+    sslcommand+=" -cipher $ciphersuite"
 
     verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'"
     # If the connection succeeded with the current cipher, benchmark and store
@@ -1095,7 +1098,8 @@ test_serverside_ordering() {
     elif [[ -e "$CACERTS" ]]; then
         sslcommand+=" -CAfile $CACERTS"
     fi
-    sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+    sslcommand+=" -trusted_first -status $SCLIENTARGS -connect $TARGET"
+    sslcommand+=" -cipher $ciphersuite"
 
     test_cipher_on_target "$sslcommand"
     if (( $? != 0 )); then
@@ -1130,6 +1134,7 @@ test_curves() {
         sslcommand+=" -CAfile $CACERTS"
     fi
     sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $current_cipher"
+    sslcommand+=" -trusted_first"
     # force the TLS to send a TLS1.0 client hello at least, as with SSLv2
     # ciphers present it will try to send a SSLv2 compatible client hello
     sslcommand+=" -no_ssl2 -no_ssl3"
@@ -1248,6 +1253,7 @@ test_curves_fallback() {
     elif [[ -e "$CACERTS" ]]; then
         sslcommand+=" -CAfile $CACERTS"
     fi
+    sslcommand+=" -trusted_first"
     sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ecc_ciphers"
     # force the TLS to send a TLS1.0 client hello at least, as with SSLv2
     # ciphers present it will try to send a SSLv2 compatible client hello
@@ -1310,7 +1316,12 @@ test_tls_tolerance() {
     tls_vers_tests['big-SSLv3']="-no_tls1_2 -no_tls1_1 -no_tls1"
 
     local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client"
-    sslcommand+=" -status -nextprotoneg 'http/1.1'"
+    if [ -n "$CAPATH" ]; then
+        sslcommand+=" -CApath $CAPATH -showcerts"
+    elif [ -e "$CACERTS" ]; then
+        sslcommand+=" -CAfile $CACERTS"
+    fi
+    sslcommand+=" -trusted_first -status -nextprotoneg 'http/1.1'"
     sslcommand+=" $SCLIENTARGS -connect $TARGET -cipher $CIPHERSUITE"
 
     for version in "${!tls_vers_tests[@]}"; do
@@ -1350,7 +1361,7 @@ test_tls_tolerance() {
         elif [[ -e "$CACERTS" ]]; then
             sslcommand+=" -CAfile $CACERTS"
         fi
-        sslcommand+=" -connect $TARGET -cipher $CIPHERSUITE"
+        sslcommand+=" -trusted_first -connect $TARGET -cipher $CIPHERSUITE"
 
         ratelimit
         verbose "Testing fallback with $sslcommand"
@@ -1375,7 +1386,7 @@ test_tls_tolerance() {
         elif [[ -e "$CACERTS" ]]; then
             sslcommand+=" -CAfile $CACERTS"
         fi
-        sslcommand+=" -connect $TARGET -cipher $ciphers"
+        sslcommand+=" -trusted_first -connect $TARGET -cipher $ciphers"
 
         ratelimit
         verbose "Testing fallback with $sslcommand"
@@ -1446,6 +1457,7 @@ test_tls_tolerance() {
         elif [[ -e "$CACERTS" ]]; then
             sslcommand+=" -CAfile $CACERTS"
         fi
+        sslcommand+=" -trusted_first"
         sslcommand+=" $SCLIENTARGS -connect $TARGET -cipher $ciphers:!SSLv2"
 
         ratelimit
@@ -1581,7 +1593,8 @@ test_kex_sigalgs() {
         elif [ -e "$CACERTS" ]; then
             sslcommand+=" -CAfile $CACERTS"
         fi
-        sslcommand+=" $SCLIENTARGS -connect $TARGET -cipher $supported_ecdsa_ciphers"
+        sslcommand+=" -trusted_first $SCLIENTARGS -connect $TARGET"
+        sslcommand+=" -cipher $supported_ecdsa_ciphers"
         # since some ciphers supported by server may be SSLv2 only, we need to
         # force use of TLSv1.2, otherwise openssl will send a SSLv2 compatible
         # client hello
@@ -1661,7 +1674,8 @@ test_kex_sigalgs() {
         elif [ -e "$CACERTS" ]; then
             sslcommand+=" -CAfile $CACERTS"
         fi
-        sslcommand+=" $SCLIENTARGS -connect $TARGET -cipher $supported_rsa_ciphers"
+        sslcommand+=" -trusted_first $SCLIENTARGS -connect $TARGET"
+        sslcommand+=" -cipher $supported_rsa_ciphers"
         # since some ciphers supported by server may be SSLv2 only, we need to
         # force use of TLSv1.2, otherwise openssl will send a SSLv2 compatible
         # client hello
@@ -1761,7 +1775,8 @@ test_kex_sigalgs() {
         elif [ -e "$CACERTS" ]; then
             sslcommand+=" -CAfile $CACERTS"
         fi
-        sslcommand+=" $SCLIENTARGS -connect $TARGET -cipher $supported_ecdsa_ciphers"
+        sslcommand+=" -trusted_first $SCLIENTARGS -connect $TARGET"
+        sslcommand+=" -cipher $supported_ecdsa_ciphers"
         # since some ciphers supported by server may be SSLv2 only, we need to
         # force use of TLSv1.2, otherwise openssl will send a SSLv2 compatible
         # client hello
@@ -1805,7 +1820,8 @@ test_kex_sigalgs() {
         elif [ -e "$CACERTS" ]; then
             sslcommand+=" -CAfile $CACERTS"
         fi
-        sslcommand+=" $SCLIENTARGS -connect $TARGET -cipher $supported_rsa_ciphers"
+        sslcommand+=" -trusted_first $SCLIENTARGS -connect $TARGET"
+        sslcommand+=" -cipher $supported_rsa_ciphers"
         # since some ciphers supported by server may be SSLv2 only, we need to
         # force use of TLSv1.2, otherwise openssl will send a SSLv2 compatible
         # client hello

top1m/make_ca_files.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+if [[ ${1,,} == "-h" ]] || [[ ${1,,} == "-help" ]] || \
+    [[ ${1,,} == "-help" ]] || [[ ${#} -lt 1 ]]; then
+    echo "Usage: ${0} certs_dir [target_dir]"
+    echo "Create an OpenSSL -CApath compatible dir with intermediate certificates"
+    echo "using a set of potentially untrusted intermediate CA certificates"
+    echo "and trusted, root CAs"
+    echo
+    echo " certs_dir    - Directory containing untrusted certificates to test"
+    echo "                one certificate per file, in PEM format"
+    echo " target_dir   - where to create the -CApath directory,"
+    echo "                \`ca_files' by default"
+    echo
+    echo "Script expects make_ca_trusted.sh in same directory as it is"
+    if [[ ${#} -lt 1 ]]; then
+        exit 1
+    else
+        exit 0
+    fi
+fi
+
+UNTRUSTED="$1"
+DIR="${2:-ca_files}"
+
+# search for make_ca_trusted.sh
+MAKE_CA_TRUSTED="$(dirname $0)/make_ca_trusted.sh"
+if [ ! -x $MAKE_CA_TRUSTED ]; then
+    echo "$MAKE_CA_TRUSTED not executable or missing" >&2
+fi
+
+# search for openssl
+for f in "$(dirname $0)/openssl" \
+         "$(dirname $0)/../openssl" \
+         "$(which openssl 2>/dev/null)"; do
+    if [ -x "$f" ]; then
+        OPENSSL="$f"
+        break
+    fi
+done
+if [ ! -x "$OPENSSL" ]; then
+    echo "openssl not found!" >&2
+    return 1
+fi
+
+# create a directory with initial trust anchors
+"${MAKE_CA_TRUSTED}" "${DIR}"
+
+pushd "${DIR}" >/dev/null
+
+# find CA certificates in untrusted certs directory
+unset CA_FILES
+declare -a CA_FILES
+CA_FILES=()
+for file in "${UNTRUSTED}"/*; do
+    if ${OPENSSL} x509 -in "$file" -noout -text 2>/dev/null | \
+            grep -q 'CA:TRUE'; then
+        CA_FILES+=("$file")
+    fi
+done
+echo "CA's found: ${#CA_FILES[@]}"
+
+files_added=0
+# check which CA files are actually trusted, add them to the directory
+cont="True"
+while [[ $cont == "True" ]]; do
+    cont="False"
+    for file_id in "${!CA_FILES[@]}"; do
+        file="${CA_FILES[$file_id]}"
+        # making an "untrusted" file and using it to verify certificates
+        # ends up taking much more time (6m vs 2m for 2500 certs)
+        if ${OPENSSL} verify -CApath . -trusted_first\
+                "$file" 2>/dev/null | grep -q ': OK$'; then
+
+            unset CA_FILES[$file_id]
+
+            c_sha256hash=($(${OPENSSL} x509 -in "$file" -outform DER 2>/dev/null | \
+                ${OPENSSL} dgst -sha256 -r 2>/dev/null))
+            if [ -e "${c_sha256hash}.pem" ]; then
+                continue
+            fi
+
+            cp "$file" "${c_sha256hash}.pem"
+            files_added=$((files_added+1))
+            cont="True"
+
+            c_hash=$(${OPENSSL} x509 -in "$file" -noout -hash 2>/dev/null)
+
+            for ((i=0; i<=100; i++)); do
+                if [[ ${c_hash}.$i -ef ${c_sha256hash}.pem ]]; then
+                    # already linked, skip
+                    break
+                fi
+                if [[ ! -e ${c_hash}.$i ]]; then
+                    ln -s "${c_sha256hash}.pem" "${c_hash}.$i"
+                    break
+                fi
+            done
+        fi
+    done
+done
+echo "CAs added: $files_added"
+
+popd >/dev/null

top1m/make_ca_trusted.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+if [[ ${1,,} == "-h" ]] || [[ ${1,,} == "-help" ]] || \
+    [[ ${1,,} == "--help" ]]; then
+    echo "Usage: $0 [dir_name]"
+    echo "Convert the system default trust store into an OpenSSL -CApath"
+    echo "compatible dir"
+    echo
+    echo "dir_name - name of the directory in which the certificates will be"
+    echo "           placed, \`ca_trusted' by default"
+    exit 0
+fi
+
+DIR="${1:-ca_trusted}"
+
+# sanity check the target directory
+if [[ -e $DIR ]] && [[ ! -d $DIR ]]; then
+    echo "$DIR must either not exist or be a directory!" >&2
+    exit 1
+fi
+
+# prepare directory
+if [[ ! -e $DIR ]]; then
+    mkdir "$DIR"
+    if [ $? -ne 0 ]; then
+        echo "mkdir failed, check your privileges" >&2
+        return 1
+    fi
+fi
+
+pushd "$DIR" >/dev/null
+
+# search for trust anchors
+for f in "${CACERTS}" \
+         /etc/pki/tls/certs/ca-bundle.crt \
+         /etc/ssl/certs/ca-certificates.crt \
+         "$(dirname $0)/../ca-bundle.crt" \
+         "$(dirname $0)/../../ca-bundle.crt"; do
+    if [ -e "$f" ]; then
+        CACERTS="$f"
+        break
+    fi
+done
+if [ ! -e "$CACERTS" ]; then
+    echo "No CA trust root store found" >&2
+    return 1
+fi
+
+# search for openssl
+for f in "$(dirname $0)/../openssl" \
+         "$(dirname $0)/../../openssl" \
+         "$(which openssl 2>/dev/null)"; do
+    if [ -x "$f" ]; then
+        OPENSSL="$f"
+        break
+    fi
+done
+if [ ! -x "$OPENSSL" ]; then
+    echo "openssl not found!" >&2
+    return 1
+fi
+
+# split the file with all CA certs into single certs
+awk '
+  split_after == 1 {n++;split_after=0}
+  /-----END CERTIFICATE-----/ {split_after=1}
+  {print > ".tmp.cert" n ".pem"}' < "$CACERTS"
+
+# rename the files to their sha256 hashes
+for file in .tmp.cert*.pem; do
+    f_hash=($(${OPENSSL} x509 -in "${file}" -outform der 2>/dev/null | \
+              ${OPENSSL} dgst -r -sha256 2>/dev/null))
+    f_name="${f_hash[0]}.pem"
+    mv ${file} ${f_name}
+done
+
+# create links that make the directory into -CApath compatible dir
+for file in *.pem; do
+    h=$(${OPENSSL} x509 -in "$file" -noout -hash 2>/dev/null)
+    for ((num=0; num<=100; num++)); do
+        if [[ ${h}.${num} -ef ${file} ]]; then
+            # file already linked, skip
+            break
+        fi
+        if [[ ! -e ${h}.${num} ]]; then
+            ln -s "${file}" "${h}.${num}"
+            break
+        fi
+    done
+done
+
+popd >/dev/null

top1m/parse_CAs.py

@@ -61,7 +61,8 @@ def get_path_for_hash(cert_hash):
     if not os.path.exists(f_name):
         f_name = ca_certs_path + '/' + cert_hash + '.pem'
         if not os.path.exists(f_name):
-            #print("File with hash " + c_hash + " is missing!")
+            sys.stderr.write("File with hash {0} ({1}) is missing!\n".format(
+                cert_hash, f_name))
             return None
     return f_name