diff --git a/README.md b/README.md index 651e846..808bfc9 100644 --- a/README.md +++ b/README.md @@ -44,38 +44,38 @@ Testing plain SSL/TLS: ``` linux $ ./cipherscan www.google.com:443 ................... -prio ciphersuite protocols pfs_keysize -1 ECDHE-RSA-CHACHA20-POLY1305 SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -2 ECDHE-RSA-AES128-GCM-SHA256 SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -3 ECDHE-RSA-RC4-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -4 ECDHE-RSA-AES128-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -5 AES128-GCM-SHA256 SSLv3,TLSv1,TLSv1.1,TLSv1.2 -6 RC4-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 -7 RC4-MD5 SSLv3,TLSv1,TLSv1.1,TLSv1.2 -8 ECDHE-RSA-AES256-GCM-SHA384 SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -9 ECDHE-RSA-AES256-SHA384 SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -10 ECDHE-RSA-AES256-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -11 AES256-GCM-SHA384 SSLv3,TLSv1,TLSv1.1,TLSv1.2 -12 AES256-SHA256 SSLv3,TLSv1,TLSv1.1,TLSv1.2 -13 AES256-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 -14 ECDHE-RSA-DES-CBC3-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -15 DES-CBC3-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 -16 ECDHE-RSA-AES128-SHA256 SSLv3,TLSv1,TLSv1.1,TLSv1.2 ECDH,P-256,256bits -17 AES128-SHA256 SSLv3,TLSv1,TLSv1.1,TLSv1.2 -18 AES128-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 +prio ciphersuite protocols pubkey_size signature_algorithm trusted pfs_keysize +1 ECDHE-RSA-CHACHA20-POLY1305 TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +2 ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +3 ECDHE-RSA-AES128-SHA TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +4 ECDHE-RSA-RC4-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +5 AES128-GCM-SHA256 TLSv1.2 2048 sha1WithRSAEncryption True +6 AES128-SHA256 TLSv1.2 2048 sha1WithRSAEncryption True +7 AES128-SHA TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True +8 RC4-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True +9 RC4-MD5 SSLv3,TLSv1,TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True +10 ECDHE-RSA-AES256-GCM-SHA384 TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +11 ECDHE-RSA-AES256-SHA384 TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +12 ECDHE-RSA-AES256-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +13 AES256-GCM-SHA384 TLSv1.2 2048 sha1WithRSAEncryption True +14 AES256-SHA256 TLSv1.2 2048 sha1WithRSAEncryption True +15 AES256-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True +16 ECDHE-RSA-DES-CBC3-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits +17 DES-CBC3-SHA SSLv3,TLSv1,TLSv1.1,TLSv1.2 2048 sha1WithRSAEncryption True +18 ECDHE-RSA-AES128-SHA256 TLSv1.2 2048 sha1WithRSAEncryption True ECDH,P-256,256bits ``` Testing STARTTLS: ``` darwin $ ./cipherscan -o ./openssl-mine -starttls xmpp jabber.ccc.de:5222 ......... -prio ciphersuite protocols pfs_keysize -1 DHE-RSA-AES256-SHA SSLv3,TLSv1 DH,1024bits -2 AES256-SHA SSLv3,TLSv1 -3 EDH-RSA-DES-CBC3-SHA SSLv3,TLSv1 DH,1024bits -4 DES-CBC3-SHA SSLv3,TLSv1 -5 DHE-RSA-AES128-SHA SSLv3,TLSv1 DH,1024bits -6 AES128-SHA SSLv3,TLSv1 -7 RC4-SHA SSLv3,TLSv1 -8 RC4-MD5 SSLv3,TLSv1 +prio ciphersuite protocols pubkey_size signature_algorithm trusted pfs_keysize +1 DHE-RSA-AES256-SHA SSLv3,TLSv1 2048 sha1WithRSAEncryption False DH,1024bits +2 AES256-SHA SSLv3,TLSv1 2048 sha1WithRSAEncryption False +3 EDH-RSA-DES-CBC3-SHA SSLv3,TLSv1 2048 sha1WithRSAEncryption False DH,1024bits +4 DES-CBC3-SHA SSLv3,TLSv1 2048 sha1WithRSAEncryption False +5 DHE-RSA-AES128-SHA SSLv3,TLSv1 2048 sha1WithRSAEncryption False DH,1024bits +6 AES128-SHA SSLv3,TLSv1 2048 sha1WithRSAEncryption False +7 RC4-SHA SSLv3,TLSv1 2048 sha1WithRSAEncryption False +8 RC4-MD5 SSLv3,TLSv1 2048 sha1WithRSAEncryption False ``` diff --git a/cipherscan b/cipherscan index 4d861aa..43469b2 100755 --- a/cipherscan +++ b/cipherscan @@ -8,6 +8,10 @@ DOBENCHMARK=0 BENCHMARKITER=30 OPENSSLBIN="$(dirname $0)/openssl" +CACERTS=${CACERTS:-/etc/pki/tls/certs/ca-bundle.crt} +if [ ! -e "$CACERTS" ]; then + echo "Warning: CA Certificates not found at $CACERTS, export CACERTS variable with location of your trust anchors" 1>&2 +fi CIPHERSUITE="ALL:COMPLEMENTOFALL" DEBUG=0 VERBOSE=0 @@ -64,6 +68,7 @@ test_cipher_on_target() { cipher="" protocols="" pfs="" + previous_cipher="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do debug echo \"quit\\n\" \| $sslcommand $tls_version @@ -71,10 +76,32 @@ test_cipher_on_target() { current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') + current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') + if [ -z $current_pubkey ]; then + current_pubkey=0 + fi + current_sigalg=$(openssl x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" + grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null + if [ $? -eq 0 ]; then + current_trusted="True" + else + current_trusted="False" + fi + if [ -z $current_sigalg ]; then + current_sigalg=None + fi if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then # connection failed, try again with next TLS version continue + else + verbose "connection successful; protocol: $current_protocol, cipher: $current_cipher, previous cipher: $previous_cipher" fi + # handling of TLSv1.2 only cipher suites + if [ ! -z "$previous_cipher" ] && [ "$previous_cipher" != "$current_cipher" ] && [ "$current_cipher" != "0000" ]; then + unset protocols + fi + previous_cipher=$current_cipher + # connection succeeded, add TLS version to positive results if [ -z "$protocols" ]; then protocols=$current_protocol @@ -83,6 +110,9 @@ test_cipher_on_target() { fi cipher=$current_cipher pfs=$current_pfs + pubkey=$current_pubkey + sigalg=$current_sigalg + trusted=$current_trusted # grab the cipher and PFS key size done # if cipher is empty, that means none of the TLS version worked with @@ -94,13 +124,13 @@ test_cipher_on_target() { # if cipher contains NONE, the cipher wasn't accepted elif [ "$cipher" == '(NONE) ' ]; then - result="$cipher $protocols $pfs" + result="$cipher $protocols $pubkey $sigalg $trusted $pfs" verbose "handshake failed, server returned ciphersuite '$result'" return 1 # the connection succeeded else - result="$cipher $protocols $pfs" + result="$cipher $protocols $pubkey $sigalg $trusted $pfs" verbose "handshake succeeded, server returned ciphersuite '$result'" return 0 fi @@ -133,7 +163,11 @@ bench_cipher() { get_cipher_pref() { [ "$OUTPUTFORMAT" == "terminal" ] && [ $DEBUG -lt 1 ] && echo -n '.' local ciphersuite="$1" - local sslcommand="$OPENSSLBIN s_client $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + if [ -e $CACERTS ]; then + local sslcommand="$OPENSSLBIN s_client -CAfile $CACERTS $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + else + local sslcommand="$OPENSSLBIN s_client $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + fi verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'" test_cipher_on_target "$sslcommand" local success=$? @@ -164,9 +198,9 @@ display_results_in_terminal() { done if [ $DOBENCHMARK -eq 1 ]; then - header="prio ciphersuite protocols pfs_keysize avg_handshake_microsec" + header="prio ciphersuite protocols pubkey_size signature_algoritm trusted pfs_keysize avg_handshake_microsec" else - header="prio ciphersuite protocols pfs_keysize" + header="prio ciphersuite protocols pubkey_size signature_algorithm trusted pfs_keysize" fi ctr=0 for result in "${results[@]}"; do @@ -187,7 +221,10 @@ display_results_in_json() { [ $ctr -gt 0 ] && echo -n ',' echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\"," echo -n "\"protocols\":[\"$(echo $cipher|awk '{print $2}'|sed 's/,/","/g')\"]," - pfs=$(echo $cipher|awk '{print $3}') + echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," + echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," + echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + pfs=$(echo $cipher|awk '{print $6}') [ "$pfs" == "" ] && pfs="None" echo -n "\"pfs\":\"$pfs\"}" ctr=$((ctr+1)) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 90306d2..18c67d9 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -9,10 +9,15 @@ import sys from collections import defaultdict import os +report_untrused=False + cipherstats = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) handshakestats = defaultdict(int) +keysize = defaultdict(int) +sigalg = defaultdict(int) +dsarsastack = 0 total = 0 for r,d,flist in os.walk(path): @@ -20,6 +25,10 @@ for r,d,flist in os.walk(path): """ initialize variables for stats of the current site """ temppfsstats = {} + tempkeystats = {} + tempecckeystats = {} + tempdsakeystats = {} + tempsigstats = {} ciphertypes = 0 AESGCM = False AES = False @@ -34,6 +43,9 @@ for r,d,flist in os.walk(path): TLS1 = False TLS1_1 = False TLS1_2 = False + dualstack = False + ECDSA = False + trusted = False """ process the file """ f_abs = os.path.join(r,f) @@ -53,6 +65,12 @@ for r,d,flist in os.walk(path): """ loop over list of ciphers """ for entry in results['ciphersuite']: + # some servers return different certificates with different + # ciphers, also we may become redirected to other server with + # different config (because over-reactive IPS) + if 'False' in entry['trusted'] and report_untrused == False: + continue; + """ store the ciphers supported """ if 'AES-GCM' in entry['cipher']: if not AESGCM: @@ -87,6 +105,25 @@ for r,d,flist in os.walk(path): DHE = True temppfsstats[entry['pfs']] = 1 + """ save the key size """ + if 'ECDSA' in entry['cipher']: + ECDSA = True + tempecckeystats[entry['pubkey'][0]] = 1 + elif 'DSS' in entry['cipher']: + tempdsakeystats[entry['pubkey'][0]] = 1 + elif 'AECDH' in entry['cipher'] or 'ADH' in entry['cipher']: + """ skip """ + else: + tempkeystats[entry['pubkey'][0]] = 1 + if ECDSA: + dualstack = True + + if 'True' in entry['trusted'] and not 'ADH' in entry['cipher'] and not 'AECDH' in entry['cipher']: + trusted = True + + """ save key signatures size """ + tempsigstats[entry['sigalg'][0]] = 1 + """ store the versions of TLS supported """ for protocol in entry['protocols']: if protocol == 'SSLv2': @@ -101,6 +138,10 @@ for r,d,flist in os.walk(path): TLS1_2 = True json_file.close() + """ don't store stats from unusued servers """ + if report_untrused == False and trusted == False: + continue + """ done with this file, storing the stats """ if DHE or ECDHE: pfsstats['Support PFS'] += 1 @@ -109,6 +150,19 @@ for r,d,flist in os.walk(path): for s in temppfsstats: pfsstats[s] += 1 + for s in tempkeystats: + keysize['RSA ' + s] += 1 + for s in tempecckeystats: + keysize['ECDSA ' + s] += 1 + for s in tempdsakeystats: + keysize['DSA ' + s] += 1 + + if dualstack: + dsarsastack += 1 + + for s in tempsigstats: + sigalg[s] += 1 + """ store cipher stats """ if AESGCM: cipherstats['AES-GCM'] += 1 @@ -168,6 +222,10 @@ for r,d,flist in os.walk(path): # break print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total) +if report_untrused == False: + print("Stats only from connections that did provide valid certificates") + print("(or anonymous DH from servers that do also have valid certificate installed)\n") + """ Display stats """ print("\nSupported Ciphers Count Percent") print("-------------------------+---------+-------") @@ -192,6 +250,20 @@ for stat in sorted(pfsstats): pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n") +print("\nCertificate sig alg Count Percent ") +print("-------------------------+---------+--------") +for stat in sorted(sigalg): + percent = round(sigalg[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(sigalg[stat]).ljust(10) + str(percent).ljust(9) + "\n") + +print("\nCertificate key size Count Percent ") +print("-------------------------+---------+--------") +for stat in sorted(keysize): + percent = round(keysize[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(9) + "\n") + +sys.stdout.write("RSA/ECDSA Dual Stack".ljust(25) + " " + str(dsarsastack).ljust(10) + str(round(dsarsastack/total * 100, 4)) + "\n") + print("\nSupported Protocols Count Percent") print("-------------------------+---------+-------") for stat in sorted(protocolstats): diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index 52f504b..71b1b0f 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -1,14 +1,37 @@ #!/usr/bin/env bash parallel=50 +max_bg=400 [ ! -e "results" ] && mkdir results -i=1 -while [ $i -lt 1000000 ] + +function wait_for_jobs() { + local no_jobs + no_jobs=$(jobs | wc -l) + + while [ $no_jobs -gt $1 ]; do + sleep 1 + no_jobs=$(jobs | wc -l) + done +} + +i=0 +count=$(wc -l top-1m.csv | awk '{print $1}') +while [ $i -lt $count ] do echo processings sites $i to $((i + parallel)) - for t in $(tail -$((1000000 - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2) + for t in $(tail -$(($count - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2) do - (tcping -u 10000000 $t 443; if [ $? -gt 0 ];then continue;fi;../cipherscan $t:443 -json > results/$t )& + (tcping -u 10000000 $t 443; + if [ $? -gt 0 ];then + tcping -u 10000000 www.$t 443; + if [ $? -gt 0 ]; then + continue; + else + ../cipherscan -json www.$t:443 > results/www.$t + continue; + fi; + fi;../cipherscan -json $t:443 > results/$t )& done - sleep 7 i=$(( i + parallel)) + wait_for_jobs $max_bg done +wait