Merge pull request #18 from tomato42/wip

Hodgepodge of fixes

cipherscan

@@ -9,10 +9,13 @@ DOBENCHMARK=0
 BENCHMARKITER=30
 OPENSSLBIN="$(dirname $0)/openssl"
 CACERTS=${CACERTS:-/etc/pki/tls/certs/ca-bundle.crt}
+if [ ! -e "$CACERTS" ]; then
+    CACERTS="/etc/ssl/certs/ca-certificates.crt"
+fi
 if [ ! -e "$CACERTS" ]; then
     echo "Warning: CA Certificates not found at $CACERTS, export CACERTS variable with location of your trust anchors" 1>&2
 fi
-CIPHERSUITE="ALL:COMPLEMENTOFALL"
+CIPHERSUITE="ALL:COMPLEMENTOFALL:+aRSA"
 DEBUG=0
 VERBOSE=0
 DELAY=0
@@ -75,6 +78,13 @@ test_cipher_on_target() {
     do
         debug echo \"Q\" \| $sslcommand $tls_version
         local tmp=$(echo "Q" | $sslcommand $tls_version 1>/dev/stdout 2>/dev/null)
+        if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then
+            current_ocspstaple="True"
+        else
+            current_ocspstaple="False"
+        fi
+        # filter out the OCSP server certificate
+        tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp")
         current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}')
         current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}')
         current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}')
@@ -82,7 +92,11 @@ test_cipher_on_target() {
         if [ -z $current_pubkey ]; then
             current_pubkey=0
         fi
-        current_sigalg=$(openssl x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
+        current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }')
+        if [ -z $current_tickethint ]; then
+            current_tickethint=None
+        fi
+        current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
         grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null
         if [ $? -eq 0 ]; then
             current_trusted="True"
@@ -115,6 +129,8 @@ test_cipher_on_target() {
         pubkey=$current_pubkey
         sigalg=$current_sigalg
         trusted=$current_trusted
+        tickethint=$current_tickethint
+        ocspstaple=$current_ocspstaple
         # grab the cipher and PFS key size
     done
     # if cipher is empty, that means none of the TLS version worked with
@@ -126,13 +142,13 @@ test_cipher_on_target() {
 
     # if cipher contains NONE, the cipher wasn't accepted
     elif [ "$cipher" == '(NONE)  ' ]; then
-        result="$cipher $protocols $pubkey $sigalg $trusted $pfs"
+        result="$cipher $protocols $pubkey $sigalg $trusted $tickethint $ocspstaple $pfs"
         verbose "handshake failed, server returned ciphersuite '$result'"
         return 1
 
     # the connection succeeded
     else
-        result="$cipher $protocols $pubkey $sigalg $trusted $pfs"
+        result="$cipher $protocols $pubkey $sigalg $trusted $tickethint $ocspstaple $pfs"
         verbose "handshake succeeded, server returned ciphersuite '$result'"
         return 0
     fi
@@ -166,9 +182,9 @@ get_cipher_pref() {
     [ "$OUTPUTFORMAT" == "terminal" ] && [ $DEBUG -lt 1 ] && echo -n '.'
     local ciphersuite="$1"
     if [ -e $CACERTS ]; then
-        local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+        local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
     else
-        local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+        local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
     fi
     verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'"
     test_cipher_on_target "$sslcommand"
@@ -190,6 +206,8 @@ display_results_in_terminal() {
     local pubkey
     local sigalg
     local trusted
+    local tickethint
+    local ocspstaple
     local different=False
     for cipher in "${cipherspref[@]}"; do
         pciph=$(echo $cipher|awk '{print $1}')
@@ -203,6 +221,8 @@ display_results_in_terminal() {
             pubkey=$(awk '{print $3}' <<<$cipher)
             sigalg=$(awk '{print $4}' <<<$cipher)
             trusted=$(awk '{print $5}' <<<$cipher)
+            tickethint=$(awk '{print $6}' <<<$cipher)
+            ocspstaple=$(awk '{print $7}' <<<$cipher)
         else
             if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then
                 different=True
@@ -213,6 +233,9 @@ display_results_in_terminal() {
             if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then
                 different=True
             fi
+            if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then
+                different=True
+            fi
         fi
         results=("${results[@]}" "$r")
         ctr=$((ctr+1))
@@ -220,13 +243,13 @@ display_results_in_terminal() {
 
     if [ $DOBENCHMARK -eq 1 ]; then
         if [ $different == "True" ]; then
-            header="prio ciphersuite protocols pubkey_size signature_algoritm trusted pfs_keysize avg_handshake_microsec"
+            header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint pfs_keysize avg_handshake_microsec"
         else
             header="prio ciphersuite protocols pfs_keysize avg_handshake_microsec"
         fi
     else
         if [ $different == "True" ]; then
-            header="prio ciphersuite protocols pubkey_size signature_algorithm trusted pfs_keysize"
+            header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint pfs_keysize"
         else
             header="prio ciphersuite protocols pfs_keysize"
         fi
@@ -240,7 +263,8 @@ display_results_in_terminal() {
         if [ $different == "True" ]; then
             echo $result|grep -v '(NONE)'
         else
-            echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $7}'
+            # prints priority, ciphersuite, protocols and pfs_keysize
+            echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}'
         fi
     done|column -t
     echo
@@ -250,6 +274,12 @@ display_results_in_terminal() {
         else
             echo "Certificate: UNTRUSTED, $pubkey bit, $sigalg signature"
         fi
+        echo "TLS ticket lifetime hint: $tickethint"
+    fi
+    if [[ $ocspstaple == "True" ]]; then
+        echo "OCSP stapling: supported"
+    else
+        echo "OCSP stapling: not supported"
     fi
 }
 
@@ -265,7 +295,9 @@ display_results_in_json() {
         echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"],"
         echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"],"
         echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\","
-        pfs=$(echo $cipher|awk '{print $6}')
+        echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\","
+        echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\","
+        pfs=$(echo $cipher|awk '{print $8}')
         [ "$pfs" == "" ] && pfs="None"
         echo -n "\"pfs\":\"$pfs\"}"
         ctr=$((ctr+1))

top1m/parse_results.py

@@ -23,6 +23,8 @@ protocolstats = defaultdict(int)
 handshakestats = defaultdict(int)
 keysize = defaultdict(int)
 sigalg = defaultdict(int)
+tickethint = defaultdict(int)
+ocspstaple = defaultdict(int)
 dsarsastack = 0
 total = 0
 for r,d,flist in os.walk(path):
@@ -35,6 +37,8 @@ for r,d,flist in os.walk(path):
         tempecckeystats = {}
         tempdsakeystats = {}
         tempsigstats = {}
+        tempticketstats = {}
+        tempcipherstats = {}
         ciphertypes = 0
         AESGCM = False
         AES = False
@@ -42,9 +46,13 @@ for r,d,flist in os.walk(path):
         DES3 = False
         CAMELLIA = False
         RC4 = False
+        ADH = False
         DHE = False
+        AECDH = False
         ECDHE = False
         RSA = False
+        ECDH = False
+        DH = False
         SSL2 = False
         SSL3 = False
         TLS1 = False
@@ -53,6 +61,7 @@ for r,d,flist in os.walk(path):
         dualstack = False
         ECDSA = False
         trusted = False
+        ocsp_stapling = None
 
         """ process the file """
         f_abs = os.path.join(r,f)
@@ -77,7 +86,11 @@ for r,d,flist in os.walk(path):
                     continue
 
                 """ store the ciphers supported """
-                if 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']:
+                if 'ADH' in entry['cipher'] or 'AECDH' in entry['cipher']:
+                    ciphertypes += 1
+                    name = "z:" + entry['cipher']
+                    tempcipherstats[name] = 1
+                elif 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']:
                     if not AESGCM:
                         AESGCM = True
                         ciphertypes += 1
@@ -104,15 +117,25 @@ for r,d,flist in os.walk(path):
                 else:
                     ciphertypes += 1
                     name = "z:" + entry['cipher']
-                    cipherstats[name] += 1
+                    tempcipherstats[name] = 1
 
                 """ store key handshake methods """
                 if 'ECDHE' in entry['cipher']:
                     ECDHE = True
                     temppfsstats[entry['pfs']] = 1
-                elif 'DHE' in entry['cipher']:
+                elif 'DHE' in entry['cipher'] or 'EDH' in entry['cipher']:
                     DHE = True
                     temppfsstats[entry['pfs']] = 1
+                elif 'AECDH' in entry['cipher']:
+                    AECDH = True
+                elif 'ADH' in entry['cipher']:
+                    ADH = True
+                elif 'ECDH' in entry['cipher']:
+                    ECDH = True
+                elif 'DH' in entry['cipher']:
+                    DH = True
+                else:
+                    RSA = True
 
                 """ save the key size """
                 if 'ECDSA' in entry['cipher']:
@@ -133,6 +156,17 @@ for r,d,flist in os.walk(path):
                 """ save key signatures size """
                 tempsigstats[entry['sigalg'][0]] = 1
 
+                """ save tls ticket hint """
+                if 'ticket_hint' in entry:
+                    tempticketstats[entry['ticket_hint']] = 1
+
+                """ check if OCSP stapling is supported """
+                if 'ocsp_stapling' in entry:
+                    if entry['ocsp_stapling'] == 'True':
+                        ocsp_stapling=True
+                    else:
+                        ocsp_stapling=False
+
                 """ store the versions of TLS supported """
                 for protocol in entry['protocols']:
                     if protocol == 'SSLv2':
@@ -175,6 +209,19 @@ for r,d,flist in os.walk(path):
         for s in tempsigstats:
             sigalg[s] += 1
 
+        if len(tempticketstats) == 1:
+            for s in tempticketstats:
+                tickethint[s + " only"] += 1
+        for s in tempticketstats:
+            tickethint[s] += 1
+
+        if ocsp_stapling is None:
+            ocspstaple['Unknown'] += 1
+        elif ocsp_stapling:
+            ocspstaple['Supported'] += 1
+        else:
+            ocspstaple['Unsupported'] += 1
+
         """ store cipher stats """
         if AESGCM:
             cipherstats['AES-GCM'] += 1
@@ -209,12 +256,24 @@ for r,d,flist in os.walk(path):
                         cipherstats['RC4 forced in TLS1.1+'] += 1
                 cipherstats['RC4 Preferred'] += 1
 
+        for cipher in tempcipherstats:
+            cipherstats[cipher] += 1
 
         """ store handshake stats """
+        if AECDH:
+            handshakestats['AECDH'] += 1
+        if ADH:
+            handshakestats['ADH'] += 1
         if ECDHE:
             handshakestats['ECDHE'] += 1
         if DHE:
             handshakestats['DHE'] += 1
+        if DHE and ECDHE:
+            handshakestats['ECDHE and DHE'] += 1
+        if ECDH:
+            handshakestats['ECDH'] += 1
+        if DH:
+            handshakestats['DH'] += 1
         if RSA:
             handshakestats['RSA'] += 1
 
@@ -279,6 +338,12 @@ for stat in sorted(pfsstats):
         pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4)
     sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n")
 
+print("\nTLS session ticket hint   Count     Percent ")
+print("-------------------------+---------+--------")
+for stat in sorted(tickethint):
+    percent = round(tickethint[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(tickethint[stat]).ljust(10) + str(percent).ljust(9) + "\n")
+
 print("\nCertificate sig alg     Count     Percent ")
 print("-------------------------+---------+--------")
 for stat in sorted(sigalg):
@@ -293,6 +358,12 @@ for stat in sorted(keysize):
 
 sys.stdout.write("RSA/ECDSA Dual Stack".ljust(25) + " " + str(dsarsastack).ljust(10) + str(round(dsarsastack/total * 100, 4)) + "\n")
 
+print("\nOCSP stapling             Count     Percent ")
+print("-------------------------+---------+--------")
+for stat in sorted(ocspstaple):
+    percent = round(ocspstaple[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(ocspstaple[stat]).ljust(10) + str(percent).ljust(9) + "\n")
+
 print("\nSupported Protocols       Count     Percent")
 print("-------------------------+---------+-------")
 for stat in sorted(protocolstats):

top1m/testtop1m.sh

@@ -1,35 +1,94 @@
 #!/usr/bin/env bash
-parallel=50
+parallel=10
-max_bg=400
+max_bg=50
+absolute_max_bg=100
+max_load=50
+
+if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then
+    echo "max user processes too low, use ulimit -u to increase"
+    exit 1
+fi
 [ ! -e "results" ] && mkdir results
 
 function wait_for_jobs() {
     local no_jobs
     no_jobs=$(jobs | wc -l)
 
-    while [ $no_jobs -gt $1 ]; do
+    while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do
+        if awk -v maxload=$max_load '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then
+            return
+        fi
         sleep 1
         no_jobs=$(jobs | wc -l)
     done
 }
 
+function scan_host() {
+    # do not scan the same host multiple times
+    if [ -e results/$1@$2 ]; then
+        return
+    fi
+    tcping -u 10000000 $2 443;
+    if [ $? -gt 0 ]; then
+        return
+    fi
+    ../cipherscan -json -servername $1 $2:443 > results/$1@$2
+}
+
+function scan_host_no_sni() {
+    # do not scan the same host multiple times
+    if [ -e results/$1 ]; then
+        return
+    fi
+    tcping -u 10000000 $1 443;
+    if [ $? -gt 0 ]; then
+        return
+    fi
+    ../cipherscan -json $1:443 > results/$1
+}
+
+function scan_hostname() {
+    if [[ ! -z $(awk -F. '$1>=0 && $1<=255 && $2>=0 && $2<=255 &&
+        $3>=0 && $3<=255 && $4>=0 && $4<=255 && NF==4' <<<"$1") ]]; then
+        scan_host_no_sni $1
+        return
+    fi
+
+    local host_ips=$(host $1 | awk '/has address/ {print $4}')
+    local www_ips=$(host www.$1 | awk '/has address/ {print $4}')
+    if [ ! -z "$host_ips" ] && [ ! -z "$www_ips" ]; then
+        # list of IPs that are in www but not in host
+        local diff=$(grep -Fv "$host_ips" <<< "$www_ips")
+        head -n 1 <<< "$host_ips" | while read ip; do
+            scan_host $1 $ip
+        done
+        if [ ! -z "$diff" ]; then
+            head -n 1 <<<"$diff" | while read ip; do
+                scan_host www.$1 $ip
+            done
+        fi
+    else
+        if [ ! -z "$host_ips" ]; then
+            head -n 1 <<<"$host_ips" | while read ip; do
+                scan_host $1 $ip
+            done
+        fi
+        if [ ! -z "$www_ips" ]; then
+            head -n 1 <<<"$www_ips" | while read ip; do
+                scan_host www.$1 $ip
+            done
+        fi
+    fi
+}
+
 i=0
 count=$(wc -l top-1m.csv | awk '{print $1}')
 while [ $i -lt $count ]
 do
     echo processings sites $i to $((i + parallel))
-    for t in $(tail -$(($count - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2)
+    for t in $(tail -$(($count - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2|cut -d "/" -f 1)
     do
-        (tcping -u 10000000 $t 443;
+        (scan_hostname $t)&
-         if [ $? -gt 0 ];then 
-             tcping -u 10000000 www.$t 443; 
-             if [ $? -gt 0 ]; then 
-                 continue; 
-             else 
-                 ../cipherscan -json www.$t:443 > results/www.$t
-                 continue;
-             fi;
-         fi;../cipherscan -json $t:443 > results/$t )&
     done
     i=$(( i + parallel))
     wait_for_jobs $max_bg