Merge pull request #33 from tomato42/cipherscan-fixes-3

Cipherscan fixes, speedups and saving of certificates (v3)

cipherscan

@@ -47,6 +47,11 @@ TIMEOUT=30
 # place where to put the found intermediate CA certificates and where
 # trust anchors are stored
 CAPATH=""
+SAVECRT=""
+unset known_certs
+declare -A known_certs
+unset cert_checksums
+declare -A cert_checksums
 
 # because running external commands like sleep incurs a fork penalty, we
 # first check if it is necessary
@@ -57,7 +62,9 @@ ratelimit() {
 }
 
 usage() {
-    echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
+    echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory]
+[--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json]
+[-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
     usage: $0 -h|--help
 
 $0 attempts to connect to a target site using all the ciphersuites it knows.
@@ -71,12 +78,14 @@ Use one of the options below:
 
 -a | --allciphers   Test all known ciphers individually at the end.
 -b | --benchmark    Activate benchmark mode.
---capath            use CAs from directory
+--capath            use CAs from directory (must be in OpenSSL CAdir format)
+--saveca            save intermediate certificates in CA directory
 -d | --delay        Pause for n seconds between connections
 -D | --debug        Output ALL the information.
 -h | --help         Shows this help text.
 -j | --json         Output results in JSON format.
 -o | --openssl      path/to/your/openssl binary you want to use.
+--savecrt           path where to save untrusted and leaf certificates
 -v | --verbose      Increase verbosity.
 
 The rest of the arguments will be interpreted as openssl s_client argument.
@@ -99,6 +108,123 @@ debug(){
     fi
 }
 
+c_hash() {
+    local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null)
+    for ((num=0; num<=100; num++)) ; do
+        if [[ $1/${h}.${num} -ef $2 ]]; then
+            # file already linked, ignore
+            break
+        fi
+        if [[ ! -e $1/${h}.${num} ]]; then
+            # file doesn't exist, create a link
+            pushd "$1" > /dev/null
+            ln -s "$2" "${h}.${num}"
+            popd > /dev/null
+            break
+        fi
+    done
+}
+
+parse_openssl_output() {
+    # clear variables in case matching doesn't hit them
+    current_ocspstaple="False"
+    current_cipher=""
+    current_pfs=""
+    current_protocol=""
+    current_tickethint="None"
+    current_pubkey=0
+    current_trusted="False"
+    current_sigalg="None"
+
+    certs_found=0
+    current_raw_certificates=()
+
+    while read line; do
+        # check if there isn't OCSP response data (response and responder cert)
+        if [[ $line =~ ^====================================== ]]; then
+            while read data; do
+                # check if there is a OCSP response in output
+                if [[ $data =~ OCSP\ Response\ Data ]]; then
+                    current_ocspstaple="True"
+                    continue
+                fi
+
+                # skip all data from a OCSP response
+                if [[ $data =~ ^====================================== ]]; then
+                    break
+                fi
+            done
+            continue
+        fi
+
+        # extract selected cipher
+        if [[ $line =~ New,\  ]]; then
+            local match=($line)
+            current_cipher="${match[4]}"
+            continue
+        fi
+
+        # extract data about selected temporary key
+        if [[ $line =~ Server\ Temp\ Key ]]; then
+            local match=($line)
+            current_pfs="${match[3]}${match[4]}${match[5]}${match[6]}"
+            continue
+        fi
+
+        # extract used protocol
+        if [[ $line =~ ^Protocol\ + ]]; then
+            local match=($line)
+            current_protocol="${match[2]}"
+            continue
+        fi
+
+        # extract session ticket hint
+        if [[ $line =~ ticket\ lifetime\ hint ]]; then
+            local match=($line)
+            current_tickethint="${match[5]}"
+            continue
+        fi
+
+        # extract size of server public key
+        if [[ $line =~ Server\ public\ key\ is\  ]]; then
+            local match=($line)
+            current_pubkey="${match[4]}"
+            continue
+        fi
+
+        # check if connection used trused certificate
+        if [[ $line =~ Verify\ return\ code:\ 0 ]]; then
+            current_trusted="True"
+            continue
+        fi
+
+        # extract certificates
+        if [[ $line =~ -----BEGIN\ CERTIFICATE----- ]]; then
+            current_raw_certificates[$certs_found]="$line"$'\n'
+            while read data; do
+                current_raw_certificates[$certs_found]+="$data"$'\n'
+                if [[ $data =~ -----END\ CERTIFICATE----- ]]; then
+                    break
+                fi
+            done
+            certs_found=$((certs_found+1))
+            continue
+        fi
+    done
+
+    # if we found any certs in output, process the first one and extract
+    # the signature algorithm on it (it's the server's certificate)
+    if [[ $certs_found -gt 0 ]]; then
+        local ossl_out=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"${current_raw_certificates[0]}")
+        while read data; do
+            if [[ $data =~ Signature\ Algorithm ]]; then
+                local match=($data)
+                current_sigalg="${match[2]}"
+            fi
+        done <<<"$ossl_out"
+    fi
+}
+
 # Connect to a target host with the selected ciphersuite
 test_cipher_on_target() {
     local sslcommand=$@
@@ -107,51 +233,102 @@ test_cipher_on_target() {
     protocols=""
     pfs=""
     previous_cipher=""
+    certificates=""
     for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2"
     do
         # sslv2 client hello doesn't support SNI extension
         # in SSLv3 mode OpenSSL just ignores the setting so it's ok
         # -status exception is ignored in SSLv2, go figure
         if [ "$tls_version" == "-ssl2" ]; then
-            cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand)
+            if [[ "$sslcommand" =~ (.*)(-servername\ [^ ]*)(.*) ]]; then
+                cmnd="${BASH_REMATCH[1]} ${BASH_REMATCH[3]}"
+            else
+                cmnd="$sslcommand"
+            fi
         else
             cmnd=$sslcommand
         fi
         ratelimit
         debug echo \"Q\" \| $cmnd $tls_version
         local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null)
-        if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then
-            current_ocspstaple="True"
-        else
-            current_ocspstaple="False"
-        fi
-        # filter out the OCSP server certificate
-        tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp")
 
-        # session metadata
+        parse_openssl_output <<<"$tmp"
-        current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}')
+        verbose "selected cipher is '$current_cipher'"
-        current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}')
+        verbose "using protocol '$current_protocol'"
-        current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}')
+
-        current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }')
+        # collect certificate data
-        if [ -z $current_tickethint ]; then
+        current_certificates=""
-            current_tickethint=None
+        local certificate_count=$certs_found
+        debug "server presented $certificate_count certificates"
+        local i
+        for ((i=0; i<$certificate_count; i=i+1 )); do
+
+            # extract i'th certificate
+            local cert="${current_raw_certificates[$i]}"
+            # put the output to an array instead running awk '{print $1}'
+            local cksum=($(cksum <<<"$cert"))
+            # compare the values not just checksums so that eventual collision
+            # doesn't mess up results
+            if [[ ${known_certs[$cksum]} == $cert ]]; then
+                if [ -n "${current_certificates}" ]; then
+                    current_certificates+=","
+                fi
+                current_certificates+="\"${cert_checksums[$cksum]}\""
+                continue
             fi
 
-        # certificate metadata
+            # compute sha256 fingerprint of the certificate
-        current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}')
+            local sha256sum=($(${OPENSSLBIN} x509 -outform DER\
-        if [ -z $current_pubkey ]; then
+                <<<"$cert" 2>/dev/null |\
-            current_pubkey=0
+                ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null))
+
+            # check if it is a CA certificate
+            local isCA="False"
+            if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\
+                grep 'CA:TRUE' >/dev/null; then
+                isCA="True"
             fi
-        current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
+
-        grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null
+            # build trust source for certificate verification
-        if [ $? -eq 0 ]; then
+            local trust_source=()
-            current_trusted="True"
+            if [[ -n $CAPATH ]]; then
-        else
+                trust_source=("-CApath" "$CAPATH")
-            current_trusted="False"
+            elif [[ -e $CACERTS ]]; then
+                trust_source=("-CAfile" "$CACERTS")
             fi
-        if [ -z $current_sigalg ]; then
+
-            current_sigalg=None
+            # check if the certificate is actually trusted (server may present
+            # unrelated certificates that are not trusted (including self
+            # signed ones)
+            local saved="False"
+            if ${OPENSSLBIN} verify "${trust_source[@]}" \
+                -untrusted <(printf "%s" "${current_raw_certificates[@]}") <(echo "$cert") 2>/dev/null | \
+                grep ': OK$' >/dev/null; then
+
+                # if the certificate is an intermediate CA it may be useful
+                # for connecting to servers that are misconfigured so save it
+                if [[ -n $CAPATH ]] && [[ $SAVECA == "True" ]] && [[ $isCA == "True" ]]; then
+                    if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then
+                        echo "$cert" > "$CAPATH/${sha256sum}.pem"
+                        c_hash "$CAPATH" "${sha256sum}.pem"
                     fi
+                    saved="True"
+                fi
+            fi
+            if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then
+                if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then
+                    echo "$cert" > "$SAVECRT/${sha256sum}.pem"
+                fi
+            fi
+            # save the sha sum for reporting
+            if [ -n "${current_certificates}" ]; then
+                current_certificates+=","
+            fi
+            current_certificates+="\"${sha256sum}\""
+            known_certs[$cksum]="$cert"
+            cert_checksums[$cksum]="$sha256sum"
+        done
+        debug "current_certificates: $current_certificates"
 
         # parsing finished, report result
         if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then
@@ -179,6 +356,7 @@ test_cipher_on_target() {
         trusted=$current_trusted
         tickethint=$current_tickethint
         ocspstaple=$current_ocspstaple
+        certificates="$current_certificates"
         # grab the cipher and PFS key size
     done
     # if cipher is empty, that means none of the TLS version worked with
@@ -230,7 +408,7 @@ get_cipher_pref() {
 
     local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client"
     if [ -n "$CAPATH" ]; then
-        sslcommand+=" -CApath $CAPATH"
+        sslcommand+=" -CApath $CAPATH -showcerts"
     elif [ -e $CACERTS ]; then
         sslcommand+=" -CAfile $CACERTS"
     fi
@@ -242,7 +420,8 @@ get_cipher_pref() {
     # If the connection succeeded with the current cipher, benchmark and store
     if [ $success -eq 0 ]; then
         cipherspref=("${cipherspref[@]}" "$result")
-        pciph=$(echo $result|awk '{print $1}')
+        ciphercertificates=("${ciphercertificates[@]}" "$certificates")
+        pciph=($result)
         get_cipher_pref "!$pciph:$ciphersuite"
         return 0
     fi
@@ -259,30 +438,32 @@ display_results_in_terminal() {
     local different=False
     echo "Target: $TARGET"; echo
     for cipher in "${cipherspref[@]}"; do
-        pciph=$(echo $cipher|awk '{print $1}')
+        # get first in array
+        pciph=($cipher)
         if [ $DOBENCHMARK -eq 1 ]; then
             bench_cipher "$pciph"
             r="$ctr $cipher $cipherbenchms"
         else
             r="$ctr $cipher"
         fi
+        local cipher_data=($cipher)
         if [ $ctr -eq 1 ]; then
-            pubkey=$(awk '{print $3}' <<<$cipher)
+            pubkey="${cipher_data[2]}"
-            sigalg=$(awk '{print $4}' <<<$cipher)
+            sigalg="${cipher_data[3]}"
-            trusted=$(awk '{print $5}' <<<$cipher)
+            trusted="${cipher_data[4]}"
-            tickethint=$(awk '{print $6}' <<<$cipher)
+            tickethint="${cipher_data[5]}"
-            ocspstaple=$(awk '{print $7}' <<<$cipher)
+            ocspstaple="${cipher_data[6]}"
         else
-            if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then
+            if [ "$pubkey" != "${cipher_data[2]}" ]; then
                 different=True
             fi
-            if [ "$sigalg" != "$(awk '{print $4}' <<<$cipher)" ]; then
+            if [ "$sigalg" != "${cipher_data[3]}" ]; then
                 different=True
             fi
-            if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then
+            if [ "$trusted" != "${cipher_data[4]}" ]; then
                 different=True
             fi
-            if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then
+            if [ "$tickethint" != "${cipher_data[5]}" ]; then
                 different=True
             fi
         fi
@@ -313,7 +494,7 @@ display_results_in_terminal() {
             echo $result|grep -v '(NONE)'
         else
             # prints priority, ciphersuite, protocols and pfs_keysize
-            echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}'
+            awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result"
         fi
     done|column -t
     echo
@@ -342,15 +523,19 @@ display_results_in_json() {
     ctr=0
     echo -n "{\"target\":\"$TARGET\",\"utctimestamp\":\"$(date -u '+%FT%T.0Z')\",\"serverside\":\"${serverside}\",\"ciphersuite\": ["
     for cipher in "${cipherspref[@]}"; do
+        local cipher_arr=($cipher)
         [ $ctr -gt 0 ] && echo -n ','
-        echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\","
+        echo -n "{\"cipher\":\"${cipher_arr[0]}\","
-        echo -n "\"protocols\":[\"$(echo $cipher|awk '{print $2}'|sed 's/,/","/g')\"],"
+        echo -n "\"protocols\":[\"${cipher_arr[1]//,/\",\"}\"],"
-        echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"],"
+        echo -n "\"pubkey\":[\"${cipher_arr[2]//,/\",\"}\"],"
-        echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"],"
+        echo -n "\"sigalg\":[\"${cipher_arr[3]//,/\",\"}\"],"
-        echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\","
+        echo -n "\"trusted\":\"${cipher_arr[4]//,/\",\"}\","
-        echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\","
+        if [[ -n $CAPATH ]]; then
-        echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\","
+            echo -n "\"certificates\":[${ciphercertificates[$ctr]}],"
-        pfs=$(echo $cipher|awk '{print $8}')
+        fi
+        echo -n "\"ticket_hint\":\"${cipher_arr[5]}\","
+        echo -n "\"ocsp_stapling\":\"${cipher_arr[6]}\","
+        pfs="${cipher_arr[7]}"
         [ "$pfs" == "" ] && pfs="None"
         echo -n "\"pfs\":\"$pfs\"}"
         ctr=$((ctr+1))
@@ -368,31 +553,31 @@ test_serverside_ordering() {
     # server supports just two ciphers, so rotate them, that should be enough
     elif [[ ${#cipherspref[@]} -eq 2 ]]; then
 
-        local cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
+        local cipher=(${cipherspref[1]})
         prefered="$cipher"
         ciphersuite=$cipher
 
-        cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
+        cipher=(${cipherspref[0]})
         ciphersuite+=":$cipher"
 
     # server supports 3 or more ciphers, rotate all three. This is necessary because google does
     # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or
     # ECDHE-RSA-CHACHA20-POLY1305
     else
-        local cipher=$(awk '{print $1}' <<< ${cipherspref[2]})
+        local cipher=(${cipherspref[2]})
         prefered="$cipher"
         ciphersuite="$cipher"
 
-        cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
+        cipher=(${cipherspref[1]})
         ciphersuite+=":$cipher"
 
-        cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
+        cipher=(${cipherspref[0]})
         ciphersuite+=":$cipher"
     fi
 
     local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client"
     if [ -n "$CAPATH" ]; then
-        sslcommand+=" -CApath $CAPATH"
+        sslcommand+=" -CApath $CAPATH -showcerts"
     elif [ -e "$CACERTS" ]; then
         sslcommand+=" -CAfile $CACERTS"
     fi
@@ -402,7 +587,7 @@ test_serverside_ordering() {
     if [ $? -ne 0 ]; then
         serverside="True"
     else
-        local selected=$(awk '{print $1}' <<< $result)
+        local selected=($result)
         if [[ $selected == $prefered ]]; then
             serverside="False"
         else
@@ -452,6 +637,14 @@ do
             CAPATH="$2"
             shift 2
             ;;
+        --saveca)
+            SAVECA="True"
+            shift 1
+            ;;
+        --savecrt)
+            SAVECRT="$2"
+            shift 2
+            ;;
         --) # End of all options
             shift
             break
@@ -491,8 +684,7 @@ TARGET=$HOST:$PORT
 debug "target: $TARGET"
 
 # test our openssl is usable
-tmp="$($OPENSSLBIN -h 2>&1 1>/dev/null)"
+if [ ! -x $OPENSSLBIN ]; then
-if [ $? -gt 0 ]; then
     OPENSSLBIN=$(which openssl)
     if [ "$OUTPUTFORMAT" == "terminal" ]; then
         echo "custom openssl not executable, falling back to system one from $OPENSSLBIN"
@@ -504,6 +696,7 @@ debug "sclientargs: $SCLIENTARGS"
 
 
 cipherspref=();
+ciphercertificates=()
 results=()
 
 # Call to the recursive loop that retrieves the cipher preferences

top1m/testtop1m.sh

@@ -9,6 +9,42 @@ if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then
     exit 1
 fi
 [ ! -e "results" ] && mkdir results
+[ ! -e "certs" ] && mkdir certs
+if [ -z "$CACERTS" ]; then
+    for f in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do
+        if [ -e "$f" ]; then
+            CACERTS="$f"
+            break
+        fi
+    done
+fi
+if [ ! -e "$CACERTS" ]; then
+  echo "file with CA certificates does not exist, please export CACERTS variable with location"
+  exit 1
+fi
+if [ ! -e "ca_files" ]; then
+    mkdir ca_files
+    pushd ca_files >/dev/null
+    awk '
+      split_after == 1 {n++;split_after=0}
+      /-----END CERTIFICATE-----/ {split_after=1}
+      {print > "cert" n ".pem"}' < "$CACERTS"
+    for i in *; do
+        h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null)
+        for num in `seq 0 100`; do
+            if [[ $h.$num -ef $i ]]; then
+                # file already linked, ignore
+                break
+            fi
+            if [[ ! -e $h.$num ]]; then
+                # file doesn't exist, create a link
+                ln -s "$i" "$h.$num"
+                break
+            fi
+        done
+    done
+    popd >/dev/null
+fi
 
 function wait_for_jobs() {
     local no_jobs
@@ -32,7 +68,7 @@ function scan_host() {
     if [ $? -gt 0 ]; then
         return
     fi
-    ../cipherscan --delay 2 -json -servername $1 $2:443 > results/$1@$2
+    ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json -servername $1 $2:443 > results/$1@$2
 }
 
 function scan_host_no_sni() {
@@ -44,7 +80,7 @@ function scan_host_no_sni() {
     if [ $? -gt 0 ]; then
         return
     fi
-    ../cipherscan --delay 2 -json $1:443 > results/$1
+    ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json $1:443 > results/$1
 }
 
 function scan_hostname() {