From 3bab7150127b55049ccbc5d18fb5a1645100d004 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Tue, 24 Jun 2014 15:34:44 +0200 Subject: [PATCH 01/28] count ECDH-RSA ciphers as ECDSA the ECDH parameters come from server certificate - the point on elliptic curve. The RSA comes from the signature on the certificate which comes from CA --- top1m/parse_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 40d85f6..4814267 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -138,7 +138,7 @@ for r,d,flist in os.walk(path): RSA = True """ save the key size """ - if 'ECDSA' in entry['cipher']: + if 'ECDSA' in entry['cipher'] or 'ECDH-RSA' in entry['cipher']: ECDSA = True tempecckeystats[entry['pubkey'][0]] = 1 elif 'DSS' in entry['cipher']: From 144e6ea2f7ea6d10c8787e3f0b81c7882986fcdc Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Tue, 24 Jun 2014 15:59:24 +0200 Subject: [PATCH 02/28] sort reported TLS session ticket hint using natural sort --- top1m/parse_results.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 4814267..6f8f500 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -14,6 +14,12 @@ import json import sys from collections import defaultdict import os +import re + +def natural_sort(l): + convert = lambda text: int(text) if text.isdigit() else text.lower() + alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] + return sorted(l, key = alphanum_key) report_untrused=False @@ -340,7 +346,7 @@ for stat in sorted(pfsstats): print("\nTLS session ticket hint Count Percent ") print("-------------------------+---------+--------") -for stat in sorted(tickethint): +for stat in natural_sort(tickethint): percent = round(tickethint[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(tickethint[stat]).ljust(10) + str(percent).ljust(9) + "\n") From 17bc04f71d808dc467f36e3cb4db42d2e6b529ea Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Wed, 25 Jun 2014 14:37:21 +0200 Subject: [PATCH 03/28] add missing ocsp_staple header --- cipherscan | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cipherscan b/cipherscan index eb91255..d703849 100755 --- a/cipherscan +++ b/cipherscan @@ -250,13 +250,13 @@ display_results_in_terminal() { if [ $DOBENCHMARK -eq 1 ]; then if [ $different == "True" ]; then - header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint pfs_keysize avg_handshake_microsec" + header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint ocsp_staple pfs_keysize avg_handshake_microsec" else header="prio ciphersuite protocols pfs_keysize avg_handshake_microsec" fi else if [ $different == "True" ]; then - header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint pfs_keysize" + header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint ocsp_staple pfs_keysize" else header="prio ciphersuite protocols pfs_keysize" fi From 4ffa061977a1be5c9d93257931da2ba547f465ee Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Wed, 25 Jun 2014 14:38:59 +0200 Subject: [PATCH 04/28] make the output shorter in case the server supports all protocol types --- cipherscan | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index d703849..24f26b3 100755 --- a/cipherscan +++ b/cipherscan @@ -268,7 +268,11 @@ display_results_in_terminal() { ctr=$((ctr+1)) fi if [ $different == "True" ]; then - echo $result|grep -v '(NONE)' + if [[ $(awk '{print $3}' <<< $result) == "SSLv3,TLSv1,TLSv1.1,TLSv1.2" ]]; then + echo $result|grep -v '(NONE)' | awk '{print $1 " " $2 " " "SSLv3-TLSv1.2" " " $4 " " $5 " " $6 " " $7 " " $8 " " $9}' + else + echo $result|grep -v '(NONE)' + fi else # prints priority, ciphersuite, protocols and pfs_keysize echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}' From 8bcc9d3cf653c59896743baaac604005a3be1fe7 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 30 Jun 2014 10:42:53 +0200 Subject: [PATCH 05/28] report ciphers causing incompatibility for Firefox It turns out that the situation is even more bleak for Firefox with regards to RC4, add it to report --- top1m/parse_results.py | 87 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 6f8f500..1a6be49 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -21,9 +21,38 @@ def natural_sort(l): alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] return sorted(l, key = alphanum_key) +""" list of ciphers offerred by Firefox 29 by default """ +firefox_ciphers=[ + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES256-SHA', + 'ECDHE-ECDSA-AES128-SHA', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-RSA-AES256-SHA', + 'ECDHE-RSA-DES-CBC3-SHA', + 'ECDHE-ECDSA-RC4-SHA', + 'ECDHE-RSA-RC4-SHA', + 'DHE-RSA-AES128-SHA', + 'DHE-DSS-AES128-SHA', + 'DHE-RSA-CAMELLIA128-SHA', + 'DHE-RSA-AES256-SHA', + 'DHE-DSS-AES256-SHA', + 'DHE-RSA-CAMELLIA256-SHA', + 'EDH-RSA-DES-CBC3-SHA', + 'AES128-SHA', + 'CAMELLIA128-SHA', + 'AES256-SHA', + 'CAMELLIA256-SHA', + 'DES-CBC3-SHA', + 'RC4-SHA', + 'RC4-MD5'] + report_untrused=False cipherstats = defaultdict(int) +FF_RC4_Only_cipherstats = defaultdict(int) +FF_RC4_preferred_cipherstats = defaultdict(int) +FF_incompatible_cipherstats = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) handshakestats = defaultdict(int) @@ -52,6 +81,11 @@ for r,d,flist in os.walk(path): DES3 = False CAMELLIA = False RC4 = False + """ the following depends on FF_compat, so by default it can be True """ + RC4_Only_FF = True + FF_compat = False + temp_FF_incompat = {} + FF_RC4_Pref = None ADH = False DHE = False AECDH = False @@ -91,6 +125,20 @@ for r,d,flist in os.walk(path): if 'False' in entry['trusted'] and report_untrused == False: continue + # check if the advertised ciphers are not effectively RC4 Only + # for firefox or incompatible with firefox + if entry['cipher'] in firefox_ciphers: + # if this is first cipher and we already are getting RC4 + # then it means that RC4 is preferred + if not FF_compat: + if 'RC4' in entry['cipher']: + FF_RC4_Pref = True + FF_compat = True + if not 'RC4' in entry['cipher']: + RC4_Only_FF = False + else: + temp_FF_incompat[entry['cipher']] = 1 + """ store the ciphers supported """ if 'ADH' in entry['cipher'] or 'AECDH' in entry['cipher']: ciphertypes += 1 @@ -262,6 +310,20 @@ for r,d,flist in os.walk(path): cipherstats['RC4 forced in TLS1.1+'] += 1 cipherstats['RC4 Preferred'] += 1 + if FF_compat: + if RC4_Only_FF and ciphertypes != 1: + cipherstats['x:FF 29 RC4 Only'] += 1 + for cipher in temp_FF_incompat: + FF_RC4_Only_cipherstats[cipher] += 1 + if FF_RC4_Pref and not 'RC4' in results['ciphersuite'][0]['cipher']: + cipherstats['x:FF 29 RC4 Preferred'] += 1 + for cipher in temp_FF_incompat: + FF_RC4_preferred_cipherstats[cipher] += 1 + else: + cipherstats['x:FF 29 incompatible'] += 1 + for cipher in temp_FF_incompat: + FF_incompatible_cipherstats[cipher] += 1 + for cipher in tempcipherstats: cipherstats[cipher] += 1 @@ -315,6 +377,13 @@ for r,d,flist in os.walk(path): #if total % 1999 == 0: # break +""" The 'x:FF 29 RC4 Preferred' counts only sites that effectively prefer + RC4 when using FF, to make reporting more readable, sum it with sites + that do that for all ciphers""" + +if "x:FF 29 RC4 Preferred" in cipherstats and "RC4 Preferred" in cipherstats: + cipherstats['x:FF 29 RC4 Preferred'] += cipherstats['RC4 Preferred'] + print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total) if report_untrused == False: print("Stats only from connections that did provide valid certificates") @@ -327,6 +396,24 @@ for stat in sorted(cipherstats): percent = round(cipherstats[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") +print("\nFF 29 RC4 Only other ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_RC4_Only_cipherstats): + percent = round(FF_RC4_Only_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_Only_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nFF 29 RC4 pref other ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_RC4_preferred_cipherstats): + percent = round(FF_RC4_preferred_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_preferred_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nFF 29 incompatible ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_incompatible_cipherstats): + percent = round(FF_incompatible_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_incompatible_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + print("\nSupported Handshakes Count Percent") print("-------------------------+---------+-------") for stat in sorted(handshakestats): From 5c4a8e8fd68fbca3ba8f3c476c2c75a2e6e14c84 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 30 Jun 2014 21:34:56 +0200 Subject: [PATCH 06/28] report what ciphers Firefox would select while connecting to server --- top1m/parse_results.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 1a6be49..ff1aa72 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -53,6 +53,7 @@ cipherstats = defaultdict(int) FF_RC4_Only_cipherstats = defaultdict(int) FF_RC4_preferred_cipherstats = defaultdict(int) FF_incompatible_cipherstats = defaultdict(int) +FF_selected_cipherstats = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) handshakestats = defaultdict(int) @@ -86,6 +87,7 @@ for r,d,flist in os.walk(path): FF_compat = False temp_FF_incompat = {} FF_RC4_Pref = None + FF_selected = None ADH = False DHE = False AECDH = False @@ -131,6 +133,7 @@ for r,d,flist in os.walk(path): # if this is first cipher and we already are getting RC4 # then it means that RC4 is preferred if not FF_compat: + FF_selected = entry['cipher'] if 'RC4' in entry['cipher']: FF_RC4_Pref = True FF_compat = True @@ -311,6 +314,13 @@ for r,d,flist in os.walk(path): cipherstats['RC4 Preferred'] += 1 if FF_compat: + if 'ECDHE' in FF_selected: + FF_selected_cipherstats['x:ECDHE'] += 1 + elif 'DHE' in FF_selected or 'EDH' in FF_selected: + FF_selected_cipherstats['x:DHE'] += 1 + else: + FF_selected_cipherstats['x:kRSA'] += 1 + FF_selected_cipherstats[FF_selected] += 1 if RC4_Only_FF and ciphertypes != 1: cipherstats['x:FF 29 RC4 Only'] += 1 for cipher in temp_FF_incompat: @@ -396,6 +406,12 @@ for stat in sorted(cipherstats): percent = round(cipherstats[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") +print("\nFF 29 selected ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_selected_cipherstats): + percent = round(FF_selected_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_selected_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + print("\nFF 29 RC4 Only other ciphers Count Percent") print("-----------------------------+---------+------") for stat in sorted(FF_RC4_Only_cipherstats): From 0ae9d767713ec5613f80db0114e58c08f03ea9b9 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 30 Jun 2014 23:03:27 +0200 Subject: [PATCH 07/28] openssl in -ssl2 mode doesn't tolerate -servername option when openssl is run in -ssl2 mode, it doesn't accept -servername option and just aborts operation, it doesn't consider -status to be special though. Remove this option when running the SSLv2 portion of the test. --- cipherscan | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cipherscan b/cipherscan index 24f26b3..ba10d14 100755 --- a/cipherscan +++ b/cipherscan @@ -78,13 +78,22 @@ debug(){ test_cipher_on_target() { local sslcommand=$@ cipher="" + local cmnd="" protocols="" pfs="" previous_cipher="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do - debug echo \"Q\" \| $sslcommand $tls_version - local tmp=$(echo "Q" | $sslcommand $tls_version 1>/dev/stdout 2>/dev/null) + # sslv2 client hello doesn't support SNI extension + # in SSLv3 mode OpenSSL just ignores the setting so it's ok + # -status exception is ignored in SSLv2, go figure + if [ "$tls_version" == "-ssl2" ]; then + cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand) + else + cmnd=$sslcommand + fi + debug echo \"Q\" \| $cmnd $tls_version + local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null) if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then current_ocspstaple="True" else From ab66f04e53d4e6ede0234e054d8f708fadedf749 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 30 Jun 2014 23:03:55 +0200 Subject: [PATCH 08/28] report if server uses client side or server side cipher ordering --- cipherscan | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index ba10d14..5573886 100755 --- a/cipherscan +++ b/cipherscan @@ -301,13 +301,18 @@ display_results_in_terminal() { else echo "OCSP stapling: not supported" fi + if [[ $serverside == "True" ]]; then + echo "Server side cipher ordering" + else + echo "Client side cipher ordering" + fi } display_results_in_json() { # Display the results in json ctr=0 - echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"ciphersuite\": [" + echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"serverside\":\"${serverside}\",\"ciphersuite\": [" for cipher in "${cipherspref[@]}"; do [ $ctr -gt 0 ] && echo -n ',' echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\"," @@ -325,6 +330,59 @@ display_results_in_json() { echo ']}' } +test_serverside_ordering() { + + local ciphersuite="" + local prefered="" + # server supports only one cipher or no ciphers, so it effectively uses server side ordering... + if [[ ${#cipherspref[@]} -lt 2 ]]; then + serverside="True" + return 0 + # server supports just two ciphers, so rotate them, that should be enough + elif [[ ${#cipherspref[@]} -eq 2 ]]; then + + local cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + prefered="$cipher" + ciphersuite=$cipher + + cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + ciphersuite+=":$cipher" + + # server supports 3 or more ciphers, rotate all three. This is necessary because google does + # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or + # ECDHE-RSA-CHACHA20-POLY1305 + else + local cipher=$(awk '{print $1}' <<< ${cipherspref[2]}) + prefered="$cipher" + ciphersuite="$cipher" + + cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + ciphersuite+=":$cipher" + + cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + ciphersuite+=":$cipher" + fi + + + if [ -e $CACERTS ]; then + local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + else + local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + fi + + test_cipher_on_target "$sslcommand" + if [ $? -ne 0 ]; then + serverside="True" + else + local selected=$(awk '{print $1}' <<< $result) + if [[ $selected == $prefered ]]; then + serverside="False" + else + serverside="True" + fi + fi +} + # UNKNOWNOPTIONS="" while : do @@ -411,6 +469,8 @@ results=() # Call to the recursive loop that retrieves the cipher preferences get_cipher_pref $CIPHERSUITE +test_serverside_ordering + if [ "$OUTPUTFORMAT" == "json" ]; then display_results_in_json else From 30d0839df63f23d9ecba07dd832cedeae494f355 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Tue, 1 Jul 2014 00:01:32 +0200 Subject: [PATCH 09/28] report cipher ordering in scanning stats, use it to simulate handshakes since now we know if server honours client order or not, we can use it to properly simulate handshakes for a given client, also report the general stats of this server configuration variable --- top1m/parse_results.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index ff1aa72..754cbb0 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -54,6 +54,7 @@ FF_RC4_Only_cipherstats = defaultdict(int) FF_RC4_preferred_cipherstats = defaultdict(int) FF_incompatible_cipherstats = defaultdict(int) FF_selected_cipherstats = defaultdict(int) +cipherordering = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) handshakestats = defaultdict(int) @@ -86,6 +87,7 @@ for r,d,flist in os.walk(path): RC4_Only_FF = True FF_compat = False temp_FF_incompat = {} + list_of_ciphers = [] FF_RC4_Pref = None FF_selected = None ADH = False @@ -127,15 +129,13 @@ for r,d,flist in os.walk(path): if 'False' in entry['trusted'] and report_untrused == False: continue + list_of_ciphers.append(entry['cipher']) + # check if the advertised ciphers are not effectively RC4 Only # for firefox or incompatible with firefox if entry['cipher'] in firefox_ciphers: # if this is first cipher and we already are getting RC4 # then it means that RC4 is preferred - if not FF_compat: - FF_selected = entry['cipher'] - if 'RC4' in entry['cipher']: - FF_RC4_Pref = True FF_compat = True if not 'RC4' in entry['cipher']: RC4_Only_FF = False @@ -263,6 +263,32 @@ for r,d,flist in os.walk(path): if dualstack: dsarsastack += 1 + """ save cipher ordering """ + if 'serverside' in results: + if results['serverside'] == "False": + cipherordering['Client side'] += 1 + else: + cipherordering['Server side'] += 1 + else: + cipherordering['Unknown'] += 1 + + """ simulate handshake with Firefox """ + if FF_compat: + if 'serverside' in results and results['serverside'] == "False": + for cipher in firefox_ciphers: + if cipher in list_of_ciphers: + FF_selected = cipher + if 'RC4' in cipher: + FF_RC4_Pref = True + break + else: + for cipher in list_of_ciphers: + if cipher in firefox_ciphers: + FF_selected = cipher + if 'RC4' in cipher: + FF_RC4_Pref = True + break + for s in tempsigstats: sigalg[s] += 1 @@ -406,6 +432,12 @@ for stat in sorted(cipherstats): percent = round(cipherstats[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") +print("\nCipher ordering Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(cipherordering): + percent = round(cipherordering[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(cipherordering[stat]).ljust(10) + str(percent).ljust(4) + "\n") + print("\nFF 29 selected ciphers Count Percent") print("-----------------------------+---------+------") for stat in sorted(FF_selected_cipherstats): From 7f5743620b0af2e43b356a019d01a8e88602c076 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 3 Jul 2014 19:02:33 +0200 Subject: [PATCH 10/28] add support for CApath capath for relatively small cert sets (~300) makes scanning about 5% faster also do a little clean up of the command-to-run generation code --- cipherscan | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/cipherscan b/cipherscan index 5573886..21fbf47 100755 --- a/cipherscan +++ b/cipherscan @@ -29,10 +29,12 @@ DELAY=0 ALLCIPHERS=0 OUTPUTFORMAT="terminal" TIMEOUT=10 - +# place where to put the found intermediate CA certificates and where +# trust anchors are stored +CAPATH="" usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -46,6 +48,7 @@ Use one of the options below: -a | --allciphers Test all known ciphers individually at the end. -b | --benchmark Activate benchmark mode. +--capath use CAs from directory -d | --delay Pause for n seconds between connections -D | --debug Output ALL the information. -h | --help Shows this help text. @@ -197,11 +200,15 @@ bench_cipher() { get_cipher_pref() { [ "$OUTPUTFORMAT" == "terminal" ] && [ $DEBUG -lt 1 ] && echo -n '.' local ciphersuite="$1" - if [ -e $CACERTS ]; then - local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" - else - local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + + local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client" + if [ -n "$CAPATH" ]; then + sslcommand+=" -CApath $CAPATH" + elif [ -e $CACERTS ]; then + sslcommand+=" -CAfile $CACERTS" fi + sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'" test_cipher_on_target "$sslcommand" local success=$? @@ -363,12 +370,13 @@ test_serverside_ordering() { ciphersuite+=":$cipher" fi - - if [ -e $CACERTS ]; then - local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" - else - local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client" + if [ -n "$CAPATH" ]; then + sslcommand+=" -CApath $CAPATH" + elif [ -e "$CACERTS" ]; then + sslcommand+=" -CAfile $CACERTS" fi + sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" test_cipher_on_target "$sslcommand" if [ $? -ne 0 ]; then @@ -420,6 +428,10 @@ do DELAY=$2 shift 2 ;; + --capath) + CAPATH="$2" + shift 2 + ;; --) # End of all options shift break From aeffc87e05a365c1d41771d72d3f8eb0745d2a0e Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 3 Jul 2014 19:09:55 +0200 Subject: [PATCH 11/28] add some comments, group related code --- cipherscan | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cipherscan b/cipherscan index 21fbf47..712de34 100755 --- a/cipherscan +++ b/cipherscan @@ -104,17 +104,21 @@ test_cipher_on_target() { fi # filter out the OCSP server certificate tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") + + # session metadata current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') - current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') - if [ -z $current_pubkey ]; then - current_pubkey=0 - fi current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }') if [ -z $current_tickethint ]; then current_tickethint=None fi + + # certificate metadata + current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') + if [ -z $current_pubkey ]; then + current_pubkey=0 + fi current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null if [ $? -eq 0 ]; then @@ -125,6 +129,8 @@ test_cipher_on_target() { if [ -z $current_sigalg ]; then current_sigalg=None fi + + # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then # connection failed, try again with next TLS version continue From 56893f7b2fadb082b2382f52a038853d2f10529c Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 4 Jul 2014 18:36:16 +0200 Subject: [PATCH 12/28] add caching of intermediate CA certificates --- cipherscan | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/cipherscan b/cipherscan index 712de34..3421c13 100755 --- a/cipherscan +++ b/cipherscan @@ -48,7 +48,7 @@ Use one of the options below: -a | --allciphers Test all known ciphers individually at the end. -b | --benchmark Activate benchmark mode. ---capath use CAs from directory +--capath use CAs from directory, save intermediate certificates there -d | --delay Pause for n seconds between connections -D | --debug Output ALL the information. -h | --help Shows this help text. @@ -77,6 +77,23 @@ debug(){ fi } +c_hash() { + local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) + for num in $(seq 0 100); do + if [[ $1/${h}.${num} -ef $2 ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $1/${h}.${num} ]]; then + # file doesn't exist, create a link + pushd "$1" > /dev/null + ln -s "$2" "${h}.${num}" + popd > /dev/null + break + fi + done +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -85,6 +102,7 @@ test_cipher_on_target() { protocols="" pfs="" previous_cipher="" + certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do # sslv2 client hello doesn't support SNI extension @@ -130,6 +148,63 @@ test_cipher_on_target() { current_sigalg=None fi + # collect certificate data + current_certificates="" + local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ + <<<"$tmp") + debug "server presented $certificate_count certificates" + local i + for ((i=0; i<$certificate_count; i=i+1 )); do + + # extract i'th certificate + local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {if (n == i) print } + ' <<<"$tmp") + + # compute sha256 fingerprint of the certificate + local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + + # check if it is a CA certificate + local isCA="False" + if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\ + grep 'CA:TRUE' >/dev/null; then + isCA="True" + fi + + # build trust source for certificate verification + local trust_source=() + if [[ -n $CAPATH ]]; then + trust_source=("-CApath" "$CAPATH") + elif [[ -e $CACERTS ]]; then + trust_source=("-CAfile" "$CACERTS") + fi + + # check if the certificate is actually trusted (server may present + # unrelated certificates that are not trusted (including self + # signed ones) + if ${OPENSSLBIN} verify "${trust_source[@]}" \ + -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + grep 'OK$' >/dev/null; then + + # if the certificate is an intermediate CA it may be useful + # for connecting to servers that are misconfigured so save it + if [[ -n $CAPATH ]] && [[ $isCA == "True" ]]; then + if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then + echo "$cert" > "$CAPATH/${sha256sum}.pem" + c_hash "$CAPATH" "${sha256sum}.pem" + fi + fi + fi + # save the sha sum for reporting + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${sha256sum}\"" + done + debug "current_certificates: $current_certificates" + # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then # connection failed, try again with next TLS version @@ -156,6 +231,7 @@ test_cipher_on_target() { trusted=$current_trusted tickethint=$current_tickethint ocspstaple=$current_ocspstaple + certificates="$current_certificates" # grab the cipher and PFS key size done # if cipher is empty, that means none of the TLS version worked with @@ -209,7 +285,7 @@ get_cipher_pref() { local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e $CACERTS ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -221,6 +297,7 @@ get_cipher_pref() { # If the connection succeeded with the current cipher, benchmark and store if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") + ciphercertificates=("${ciphercertificates[@]}" "$certificates") pciph=$(echo $result|awk '{print $1}') get_cipher_pref "!$pciph:$ciphersuite" return 0 @@ -333,6 +410,9 @@ display_results_in_json() { echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + if [[ -n $CAPATH ]]; then + echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," + fi echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," pfs=$(echo $cipher|awk '{print $8}') @@ -378,7 +458,7 @@ test_serverside_ordering() { local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e "$CACERTS" ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -482,6 +562,7 @@ debug "sclientargs: $SCLIENTARGS" cipherspref=(); +ciphercertificates=() results=() # Call to the recursive loop that retrieves the cipher preferences From 431b453e438098fe71f4e5da555b8383950b5852 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 4 Jul 2014 18:51:21 +0200 Subject: [PATCH 13/28] add ability to also save leaf certificates and untrusted ones --- cipherscan | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index 3421c13..51c326b 100755 --- a/cipherscan +++ b/cipherscan @@ -32,9 +32,10 @@ TIMEOUT=10 # place where to put the found intermediate CA certificates and where # trust anchors are stored CAPATH="" +SAVECRT="" usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [--savecrt directory] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -54,6 +55,7 @@ Use one of the options below: -h | --help Shows this help text. -j | --json Output results in JSON format. -o | --openssl path/to/your/openssl binary you want to use. +--savecrt path where to save untrusted and leaf certificates -v | --verbose Increase verbosity. The rest of the arguments will be interpreted as openssl s_client argument. @@ -184,6 +186,7 @@ test_cipher_on_target() { # check if the certificate is actually trusted (server may present # unrelated certificates that are not trusted (including self # signed ones) + local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ grep 'OK$' >/dev/null; then @@ -195,6 +198,12 @@ test_cipher_on_target() { echo "$cert" > "$CAPATH/${sha256sum}.pem" c_hash "$CAPATH" "${sha256sum}.pem" fi + saved="True" + fi + fi + if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then + if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then + echo "$cert" > "$SAVECRT/${sha256sum}.pem" fi fi # save the sha sum for reporting @@ -518,6 +527,10 @@ do CAPATH="$2" shift 2 ;; + --savecrt) + SAVECRT="$2" + shift 2 + ;; --) # End of all options shift break From 2b959f601d53538c92432ac6b933b143f7bb1ac9 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 11 Jul 2014 16:43:14 +0200 Subject: [PATCH 14/28] use CApath for certificates and store certificates CApath is about 20% faster than CAfile so use it, also save the recived certificates from the servers for later analysis (proper hostname checking, looking for certificates sharing private key, etc.) --- top1m/testtop1m.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index b2cb0ef..0f07744 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -9,6 +9,30 @@ if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then exit 1 fi [ ! -e "results" ] && mkdir results +[ ! -e "certs" ] && mkdir certs +if [ ! -e "ca_files" ]; then + mkdir ca_files + pushd ca_files >/dev/null + awk ' + split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {print > "cert" n ".pem"}' < "/etc/pki/tls/certs/ca-bundle.crt" + for i in *; do + h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null) + for num in `seq 0 100`; do + if [[ $h.$num -ef $i ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $h.$num ]]; then + # file doesn't exist, create a link + ln -s "$i" "$h.$num" + break + fi + done + done + popd >/dev/null +fi function wait_for_jobs() { local no_jobs @@ -32,7 +56,7 @@ function scan_host() { if [ $? -gt 0 ]; then return fi - ../cipherscan -json -servername $1 $2:443 > results/$1@$2 + ../cipherscan --capath ca_files --savecrt certs -json -servername $1 $2:443 > results/$1@$2 } function scan_host_no_sni() { @@ -44,7 +68,7 @@ function scan_host_no_sni() { if [ $? -gt 0 ]; then return fi - ../cipherscan -json $1:443 > results/$1 + ../cipherscan --capath ca_files --savecrt certs -json $1:443 > results/$1 } function scan_hostname() { From d2f112033dbcd1e885ca5aee58fe791ec75c5647 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 11 Jul 2014 17:29:57 +0200 Subject: [PATCH 15/28] clean up the extracted certificate the certificate extracted in the above way will contain some junk from openssl s_client output we don't want like verification status we can remove it ro reduce disk usage for saved certificates --- cipherscan | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cipherscan b/cipherscan index 51c326b..a01e2e6 100755 --- a/cipherscan +++ b/cipherscan @@ -163,6 +163,8 @@ test_cipher_on_target() { /-----END CERTIFICATE-----/ {split_after=1} {if (n == i) print } ' <<<"$tmp") + # clean up the cert from junk before BEGIN CERTIFICATE + cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) # compute sha256 fingerprint of the certificate local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ From caa534bfd7fc729fd287253ce9df5ff2493ee429 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 01:30:17 +0200 Subject: [PATCH 16/28] don't retry protocols we know don't work When connection is unsuccessful with a given protocol, don't try it again since we probably exhausted the ciphers supported by the protocol makes scanning about 10% faster --- cipherscan | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index a01e2e6..520505f 100755 --- a/cipherscan +++ b/cipherscan @@ -33,6 +33,8 @@ TIMEOUT=10 # trust anchors are stored CAPATH="" SAVECRT="" +unset ok_protocols +declare -A ok_protocols usage() { echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [--savecrt directory] [-v|--verbose] [-o|--openssl file] [openssl s_client args] @@ -107,6 +109,9 @@ test_cipher_on_target() { certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do + if [[ ${ok_protocols[$tls_version]} -eq 1 ]]; then + continue + fi # sslv2 client hello doesn't support SNI extension # in SSLv3 mode OpenSSL just ignores the setting so it's ok # -status exception is ignored in SSLv2, go figure @@ -218,6 +223,7 @@ test_cipher_on_target() { # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then + ok_protocols["$tls_version"]=1 # connection failed, try again with next TLS version continue else @@ -287,7 +293,6 @@ bench_cipher() { cipherbenchms="$((t/1000/$BENCHMARKITER))" } - # Connect to the target and retrieve the chosen cipher # recursively until the connection fails get_cipher_pref() { @@ -582,6 +587,8 @@ results=() # Call to the recursive loop that retrieves the cipher preferences get_cipher_pref $CIPHERSUITE +unset ok_protocols +declare -A ok_protocols test_serverside_ordering From ef4786f1d7e2066f7d87dad54b6ee8148531fc25 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 01:43:54 +0200 Subject: [PATCH 17/28] no need to grep the input when we're using awk awk has an inbuilt version of grep, also truncate processing as soon as we find what we're looking for --- cipherscan | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cipherscan b/cipherscan index 520505f..fabff8d 100755 --- a/cipherscan +++ b/cipherscan @@ -131,20 +131,21 @@ test_cipher_on_target() { tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") # session metadata - current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') - current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') - current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') - current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }') + current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") + current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") + current_protocol=$(awk '/^\s+Protocol\s+:/ {print $3; exit}' <<<"$tmp") + current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") if [ -z $current_tickethint ]; then current_tickethint=None fi # certificate metadata - current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') + current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") if [ -z $current_pubkey ]; then current_pubkey=0 fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" + current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ + awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null if [ $? -eq 0 ]; then current_trusted="True" From bcfe0dbae11a5eda6fd0e69c35925ec2672c5fad Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:17:52 +0200 Subject: [PATCH 18/28] don't calculate sha sums for the certificates over and over we can use cksum to calculate simple checksum much faster than with using openssl, so we can compute sums only once --- cipherscan | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cipherscan b/cipherscan index fabff8d..feb19ae 100755 --- a/cipherscan +++ b/cipherscan @@ -35,6 +35,10 @@ CAPATH="" SAVECRT="" unset ok_protocols declare -A ok_protocols +unset known_certs +declare -A known_certs +unset cert_checksums +declare -A cert_checksums usage() { echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [--savecrt directory] [-v|--verbose] [-o|--openssl file] [openssl s_client args] @@ -165,15 +169,25 @@ test_cipher_on_target() { for ((i=0; i<$certificate_count; i=i+1 )); do # extract i'th certificate - local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} - /-----END CERTIFICATE-----/ {split_after=1} - {if (n == i) print } - ' <<<"$tmp") - # clean up the cert from junk before BEGIN CERTIFICATE - cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) + local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } + /-----BEGIN CERTIFICATE-----/ { output=1 } + output==1 { if (n==i) print } + /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + # put the output to an array instead awk '{print $1}' + local cksum=($(cksum <<<"$cert")) + # compare the values not just checksums so that eventual collision + # doesn't mess up results + if [[ ${known_certs[$cksum]} == $cert ]]; then + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${cert_checksums[$cksum]}\"" + continue + fi # compute sha256 fingerprint of the certificate - local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + <<<"$cert" 2>/dev/null |\ ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') # check if it is a CA certificate @@ -219,6 +233,8 @@ test_cipher_on_target() { current_certificates+="," fi current_certificates+="\"${sha256sum}\"" + known_certs[$cksum]="$cert" + cert_checksums[$cksum]="$sha256sum" done debug "current_certificates: $current_certificates" From ab8b7945573920721118214dfbf2fd616efdd0ca Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:54:33 +0200 Subject: [PATCH 19/28] few less forks in the script again, we can use arrays and a bit advanced awk syntax to reduce the number of forks necessary to run the script --- cipherscan | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cipherscan b/cipherscan index feb19ae..693cf8a 100755 --- a/cipherscan +++ b/cipherscan @@ -173,7 +173,7 @@ test_cipher_on_target() { /-----BEGIN CERTIFICATE-----/ { output=1 } output==1 { if (n==i) print } /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") - # put the output to an array instead awk '{print $1}' + # put the output to an array instead running awk '{print $1}' local cksum=($(cksum <<<"$cert")) # compare the values not just checksums so that eventual collision # doesn't mess up results @@ -331,7 +331,7 @@ get_cipher_pref() { if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") ciphercertificates=("${ciphercertificates[@]}" "$certificates") - pciph=$(echo $result|awk '{print $1}') + pciph=($(echo $result)) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -349,7 +349,7 @@ display_results_in_terminal() { local ocspstaple local different=False for cipher in "${cipherspref[@]}"; do - pciph=$(echo $cipher|awk '{print $1}') + pciph=($(echo $cipher)) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" @@ -401,13 +401,13 @@ display_results_in_terminal() { fi if [ $different == "True" ]; then if [[ $(awk '{print $3}' <<< $result) == "SSLv3,TLSv1,TLSv1.1,TLSv1.2" ]]; then - echo $result|grep -v '(NONE)' | awk '{print $1 " " $2 " " "SSLv3-TLSv1.2" " " $4 " " $5 " " $6 " " $7 " " $8 " " $9}' + awk '!/(NONE)/{print $1 " " $2 " " "SSLv3-TLSv1.2" " " $4 " " $5 " " $6 " " $7 " " $8 " " $9}' <<<"$result" else echo $result|grep -v '(NONE)' fi else # prints priority, ciphersuite, protocols and pfs_keysize - echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}' + awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result" fi done|column -t echo From 68577f791830b44cd4fb2d446e7aa7a8a1ba120e Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 25 Jul 2014 17:49:44 +0200 Subject: [PATCH 20/28] collect statistics about found certificates --- top1m/parse_CAs.py | 346 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 top1m/parse_CAs.py diff --git a/top1m/parse_CAs.py b/top1m/parse_CAs.py new file mode 100644 index 0000000..672548a --- /dev/null +++ b/top1m/parse_CAs.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Author: Hubert Kario - 2014 + +from __future__ import division + +path = "./results/" +ca_certs_path = "./ca_files" +certs_path = "./certs" + +""" only root CAs, no cached intermediate certs """ +trust_path = "./ca_trusted" + +import json +import sys +from collections import defaultdict +import os +import re +import subprocess +from OpenSSL import crypto +from M2Crypto import X509, EVP +from m2ext import _m2ext +from m2ext import SSL +import glob + +from pprint import pprint + +# override m2ext implementation so that it is possible to provide additional +# certificates to be used during verification. Requires m2ext with a small fix +# from https://github.com/tomato42/m2ext/tree/extended_ctx_init +class Context(SSL.Context): + def validate_certificate(self, cert, chain=None): + """ + Validate a certificate using this SSL Context + """ + if chain: + ptr = chain._ptr() + else: + ptr = None + store_ctx = X509.X509_Store_Context(_m2ext.x509_store_ctx_new(), _pyfree=1) + _m2ext.x509_store_ctx_init(store_ctx.ctx, + self.get_cert_store().store, + cert.x509, ptr) + rc = _m2ext.x509_verify_cert(store_ctx.ctx) + if rc < 0: + raise SSL.SSLError("Empty context") + return rc != 0 + + +invocations = defaultdict(int) + +total = 0 +hosts = 0 +chains = defaultdict(int) +chain_len = defaultdict(int) +keysize = defaultdict(int) +root_CA = defaultdict(int) +sig_alg = defaultdict(int) +intermediate_CA = defaultdict(int) + +subject_hashes = {} +issuer_hashes = {} + +def get_cert_subject_name(cert): + subject = cert.get_subject() + commonName = None + organization = None + + for elem,val in subject.get_components(): + if elem == "CN" and commonName is None: + commonName = val + if elem == "O" and organization is None: + organization = val + + s_hash = "(" + ("%0.8X" % subject.hash()).lower() + ") " + + if commonName is not None: + return s_hash + commonName + elif organization is not None: + return s_hash + organization + else: + return s_hash + +def get_cert_hashes(path): + if path in subject_hashes: + return subject_hashes[path], issuer_hashes[path] + + with open(path) as srv_c_f: + srv_c_pem = srv_c_f.read() + + srv_c = crypto.load_certificate(crypto.FILETYPE_PEM, srv_c_pem) + + # can't make M2Crypto to output OpenSSL-compatible hashes... + subject_hash = ("%0.8X" % srv_c.get_subject().hash()).lower() + issuer_hash = ("%0.8X" % srv_c.get_issuer().hash()).lower() + + subject_hashes[path] = subject_hash + issuer_hashes[path] = issuer_hash + + return subject_hash, issuer_hash + +def gen_cert_paths(paths): + + # failsafe in case of a loop in path resolution + if len(paths) > 10: + return + + subject_hash, issuer_hash = get_cert_hashes(paths[-1]) + + if subject_hash == issuer_hash: + yield paths + else: + for ca_file in glob.glob(ca_certs_path + '/' + issuer_hash + ".*"): + for perm in gen_cert_paths(paths + [ca_file]): + if not perm in paths: + yield perm + +def is_chain_complete_f(file_names): + + stack = X509.X509_Stack() + for f_name in file_names[1:]: + cert = X509.load_cert(f_name) + stack.push(cert) + + cert = X509.load_cert(file_names[0]) + + return trusted_context.validate_certificate(cert, stack) + +def is_chain_complete(certs): + + stack = X509.X509_Stack() + + for cert in certs[1:]: + stack.push(cert) + + return trusted_context.validate_certificate(certs[0], stack) + +def is_chain_trusted(cert_hashes): + + c_hash = cert_hashes[0] + """ first check the likely option: the cert dir """ + file_name = certs_path + '/' + c_hash + '.pem' + if not os.path.exists(file_name): + """ then try the unlikely option: ca directory """ + file_name = ca_certs_path + '/' + c_hash + '.pem' + if not os.path.exists(file_name): + print "File with hash " + c_hash + " is missing!" + return False,None + + for cert_paths in gen_cert_paths([ file_name ]): + if is_chain_complete_f(cert_paths): + return True,cert_paths + + return False,None + +def get_path_for_hash(cert_hash): + f_name = certs_path + '/' + c_hash + '.pem' + if not os.path.exists(f_name): + f_name = ca_certs_path + '/' + c_hash + '.pem' + if not os.path.exists(f_name): + print "File with hash " + c_hash + " is missing!" + return None + return f_name + +def is_chain_trusted_at_all(cert_list): + certs = [] + stack = X509.X509_Stack() + + cert = cert_list[0] + + for ca in cert_list[1:]: + stack.push(ca) + + return all_CAs_context.validate_certificate(cert, stack) + +def collect_key_sizes(file_names): + + """ don't collect signature alg for the self signed root """ + with open(file_names[-1]) as cert_file: + cert_pem = cert_file.read() + + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + pubkey = cert.get_pubkey() + if pubkey.type() == crypto.TYPE_RSA: + keysize['RSA ' + str(pubkey.bits())] += 1 + elif pubkey.type() == crypto.TYPE_DSA: + keysize['DSA ' + str(pubkey.bits())] += 1 + elif pubkey.type() == 408: + keysize['ECDSA ' + str(pubkey.bits())] += 1 + else: + keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1 + + root_CA[get_cert_subject_name(cert)] += 1 + + """ exclude the self signed root and server cert from stats """ + for f_name in file_names[1:-1]: + with open(f_name) as cert_file: + cert_pem = cert_file.read() + + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + pubkey = cert.get_pubkey() + if pubkey.type() == crypto.TYPE_RSA: + keysize['RSA ' + str(pubkey.bits())] += 1 + elif pubkey.type() == crypto.TYPE_DSA: + keysize['DSA ' + str(pubkey.bits())] += 1 + elif pubkey.type() == 408: + keysize['ECDSA ' + str(pubkey.bits())] += 1 + else: + keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1 + + sig_alg[cert.get_signature_algorithm()] += 1 + + intermediate_CA[get_cert_subject_name(cert)] += 1 + +all_CAs_context = Context() +all_CAs_context.load_verify_locations(capath=ca_certs_path) +trusted_context = Context() +trusted_context.load_verify_locations(capath=trust_path) + +for r,d,flist in os.walk(path): + for f in flist: + + server_chain_trusted = False + server_chain_complete = False + server_chains = [] + chains_tested = [] + valid = True + + """ process the file """ + f_abs = os.path.join(r,f) + with open(f_abs) as json_file: + """ Keep certificates in memory for a given file """ + known_certs = {} + + """ discard files that fail to load """ + try: + results = json.load(json_file) + except ValueError: + continue + + """ discard files with empty results """ + if len(results['ciphersuite']) < 1: + continue + + valid = True + + """ loop over list of ciphers """ + for entry in results['ciphersuite']: + + """ skip entries which don't have certificate references """ + if not 'certificates' in entry: + continue + + """ skip entries for A(EC)DH suites """ + if len(entry['certificates']) < 1: + continue + + if not entry['certificates'] in chains_tested: + certs = [] + + for c_hash in entry['certificates']: + if c_hash in known_certs: + certs += [known_certs[c_hash]] + else: + cert = X509.load_cert(get_path_for_hash(c_hash)) + known_certs[c_hash] = cert + certs += [cert] + + if is_chain_trusted_at_all(certs): + ret,tmpchain = is_chain_trusted(entry['certificates']) + if ret: + server_chain_trusted = True + if not tmpchain in server_chains: + server_chains += [tmpchain] + if is_chain_complete(certs): + server_chain_complete = True + + chains_tested += [entry['certificates']] + + if server_chain_trusted: + if server_chain_complete: + chains["complete"] += 1 + else: + chains["incomplete"] += 1 + else: + chains["untrusted"] += 1 + + if valid: + hosts += 1 + + for chain in server_chains: + collect_key_sizes(chain) + chain_len[str(len(chain))] += 1 + if len(chain) == 1: + print "file with chain 1 long " + f_abs + total += 1 + +""" Display stats """ +#print "openssl invocations: " + str(invocations["openssl"]) + +print "Statistics from " + str(total) + " chains provided by " + str(hosts) + " hosts" + +print("\nServer provided chains Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(chains): + percent = round(chains[stat] / hosts * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(chains[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nTrusted chain statistics") +print("========================") + + +print("\nChain length Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(chain_len): + percent = round(chain_len[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(chain_len[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nCA Key Size Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(keysize): + percent = round(keysize[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nRoot CAs Count Percent") +print("---------------------------------------------+---------+-------") +for stat in sorted(root_CA): + percent = round(root_CA[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(45)[0:45] + " " + str(root_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nSignature algorithm Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(sig_alg): + percent = round(sig_alg[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(sig_alg[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nIntermediate CA Count Percent") +print("---------------------------------------------+---------+-------") +for stat in sorted(intermediate_CA): + percent = round(intermediate_CA[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(45)[0:45] + " " + str(intermediate_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n") From b4291236bbcebeb4d12159ae2a658ed6754c70fb Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 3 Aug 2014 01:49:52 +0200 Subject: [PATCH 21/28] add collection of LoS for the cert paths found --- top1m/parse_CAs.py | 127 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 115 insertions(+), 12 deletions(-) diff --git a/top1m/parse_CAs.py b/top1m/parse_CAs.py index 672548a..72c2b8f 100644 --- a/top1m/parse_CAs.py +++ b/top1m/parse_CAs.py @@ -57,9 +57,11 @@ hosts = 0 chains = defaultdict(int) chain_len = defaultdict(int) keysize = defaultdict(int) +keysize_per_chain = defaultdict(int) root_CA = defaultdict(int) sig_alg = defaultdict(int) intermediate_CA = defaultdict(int) +effective_security = defaultdict(int) subject_hashes = {} issuer_hashes = {} @@ -161,7 +163,7 @@ def get_path_for_hash(cert_hash): if not os.path.exists(f_name): f_name = ca_certs_path + '/' + c_hash + '.pem' if not os.path.exists(f_name): - print "File with hash " + c_hash + " is missing!" + #print "File with hash " + c_hash + " is missing!" return None return f_name @@ -176,8 +178,43 @@ def is_chain_trusted_at_all(cert_list): return all_CAs_context.validate_certificate(cert, stack) +""" convert RSA and DSA key sizes to estimated Level of security """ +def rsa_key_size_to_los(size): + if size < 760: + return 40 + elif size < 1020: + return 64 + elif size < 2040: + return 80 + elif size < 3068: + return 112 + elif size < 4094: + return 128 + elif size < 7660: + return 152 + elif size < 15300: + return 192 + else: + return 256 + +def sig_alg_to_los(name): + if 'SHA1' in name.upper(): + return 80 + elif 'SHA224' in name.upper(): + return 112 + elif 'SHA256' in name.upper(): + return 128 + elif 'SHA384' in name.upper(): + return 192 + elif 'SHA512' in name.upper(): + return 256 + else: + raise UnknownSigAlgError + def collect_key_sizes(file_names): + tmp_keysize = {} + """ don't collect signature alg for the self signed root """ with open(file_names[-1]) as cert_file: cert_pem = cert_file.read() @@ -187,12 +224,19 @@ def collect_key_sizes(file_names): pubkey = cert.get_pubkey() if pubkey.type() == crypto.TYPE_RSA: keysize['RSA ' + str(pubkey.bits())] += 1 + tmp_keysize['RSA ' + str(pubkey.bits())] = 1 + security_level = rsa_key_size_to_los(pubkey.bits()) elif pubkey.type() == crypto.TYPE_DSA: keysize['DSA ' + str(pubkey.bits())] += 1 + tmp_keysize['DSA ' + str(pubkey.bits())] = 1 + security_level = rsa_key_size_to_los(pubkey.bits()) elif pubkey.type() == 408: keysize['ECDSA ' + str(pubkey.bits())] += 1 + tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1 + security_level = pubkey.bits()/2 else: keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1 + security_level = 0 root_CA[get_cert_subject_name(cert)] += 1 @@ -206,17 +250,60 @@ def collect_key_sizes(file_names): pubkey = cert.get_pubkey() if pubkey.type() == crypto.TYPE_RSA: keysize['RSA ' + str(pubkey.bits())] += 1 + tmp_keysize['RSA ' + str(pubkey.bits())] = 1 + c_key_level = rsa_key_size_to_los(pubkey.bits()) elif pubkey.type() == crypto.TYPE_DSA: keysize['DSA ' + str(pubkey.bits())] += 1 + tmp_keysize['DSA ' + str(pubkey.bits())] = 1 + c_key_level = rsa_key_size_to_los(pubkey.bits()) elif pubkey.type() == 408: keysize['ECDSA ' + str(pubkey.bits())] += 1 + tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1 + c_key_level = pubkey.bits() / 2 else: keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1 + c_key_level = 0 + + if security_level > c_key_level: + security_level = c_key_level sig_alg[cert.get_signature_algorithm()] += 1 + c_sig_level = sig_alg_to_los(cert.get_signature_algorithm()) + if security_level > c_sig_level: + security_level = c_sig_level intermediate_CA[get_cert_subject_name(cert)] += 1 + for key_s in tmp_keysize: + keysize_per_chain[key_s] += 1 + + # XXX doesn't handle the situation in which the CA uses its certificate + # for a web server properly + with open(file_names[0]) as cert_file: + cert_pem = cert_file.read() + + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + pubkey = cert.get_pubkey() + if pubkey.type() == crypto.TYPE_RSA: + c_key_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == crypto.TYPE_DSA: + c_key_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == 408: + c_key_level = pubkey.bits() / 2 + else: + c_key_level = 0 + + if security_level > c_key_level: + security_level = c_key_level + + c_sig_level = sig_alg_to_los(cert.get_signature_algorithm()) + if security_level > c_sig_level: + security_level = c_sig_level + + effective_security[security_level] += 1 + + all_CAs_context = Context() all_CAs_context.load_verify_locations(capath=ca_certs_path) trusted_context = Context() @@ -267,7 +354,10 @@ for r,d,flist in os.walk(path): if c_hash in known_certs: certs += [known_certs[c_hash]] else: - cert = X509.load_cert(get_path_for_hash(c_hash)) + path = get_path_for_hash(c_hash) + if path is None: + continue + cert = X509.load_cert(path) known_certs[c_hash] = cert certs += [cert] @@ -285,10 +375,13 @@ for r,d,flist in os.walk(path): if server_chain_trusted: if server_chain_complete: chains["complete"] += 1 + print "complete: " + f else: chains["incomplete"] += 1 + print "incomplete: " + f else: chains["untrusted"] += 1 + print "untrusted: " + f if valid: hosts += 1 @@ -321,11 +414,27 @@ for stat in sorted(chain_len): percent = round(chain_len[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(chain_len[stat]).ljust(10) + str(percent).ljust(4) + "\n") -print("\nCA Key Size Count Percent") -print("-------------------------+---------+-------") +print("\nCA key size in chains Count") +print("-------------------------+---------") for stat in sorted(keysize): - percent = round(keysize[stat] / total * 100, 4) - sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(4) + "\n") + sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + "\n") + +print("\nChains with CA key Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(keysize_per_chain): + percent = round(keysize_per_chain[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(keysize_per_chain[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nSignature algorithm (ex. root) Count") +print("------------------------------+---------") +for stat in sorted(sig_alg): + sys.stdout.write(stat.ljust(30) + " " + str(sig_alg[stat]).ljust(10) + "\n") + +print("\nEff. host cert chain LoS Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(effective_security): + percent = round(effective_security[stat] / total * 100, 4) + sys.stdout.write(str(stat).ljust(25) + " " + str(effective_security[stat]).ljust(10) + str(percent).ljust(4) + "\n") print("\nRoot CAs Count Percent") print("---------------------------------------------+---------+-------") @@ -333,12 +442,6 @@ for stat in sorted(root_CA): percent = round(root_CA[stat] / total * 100, 4) sys.stdout.write(stat.ljust(45)[0:45] + " " + str(root_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n") -print("\nSignature algorithm Count Percent") -print("-------------------------+---------+-------") -for stat in sorted(sig_alg): - percent = round(sig_alg[stat] / total * 100, 4) - sys.stdout.write(stat.ljust(25) + " " + str(sig_alg[stat]).ljust(10) + str(percent).ljust(4) + "\n") - print("\nIntermediate CA Count Percent") print("---------------------------------------------+---------+-------") for stat in sorted(intermediate_CA): From 0591829ed125720ed201fe7a094d5d90cb59cba2 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 3 Aug 2014 02:15:36 +0200 Subject: [PATCH 22/28] application for finding the correct certificate chain process the result files and output if the cert chain inside is complete or not (that means it requires further processing) --- top1m/parse_CAs.c | 509 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 509 insertions(+) create mode 100644 top1m/parse_CAs.c diff --git a/top1m/parse_CAs.c b/top1m/parse_CAs.c new file mode 100644 index 0000000..0b15609 --- /dev/null +++ b/top1m/parse_CAs.c @@ -0,0 +1,509 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * Author: Hubert Kario - 2014 + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static char* CA_TRUSTED = "./ca_trusted"; +static char* CA_ALL = "./ca_files"; +static char* CERTS_DIR = "./certs"; + +/* SSL context that knows only about trust anchors */ +SSL_CTX *trusted_only; +/* SSL context that also has access to other CA certs */ +SSL_CTX *all_CAs; + +// load certificate from file to a OpenSSL object +X509 *load_cert(char *filename) +{ + BIO* f; + X509 *ret; + + f = BIO_new(BIO_s_file()); + BIO_read_filename(f, filename); + + ret = PEM_read_bio_X509_AUX(f, NULL, 0, NULL); + if (ret == NULL) + fprintf(stderr, "Unable to load file %s as X509 certificate\n", filename); + + BIO_free_all(f); + + return ret; +} + +// convert sha256 to a file name, if the file exists +// search in "all CAs" dir and "leaf certs" directories +char *hash_to_filename(const char *hash) +{ + char *tmp_f_name; + size_t n; + + n = strlen(hash) + 30; + + // TODO error checking + tmp_f_name = malloc(n); + + /* first check if the file is in directory with regular certs */ + // TODO error checking + snprintf(tmp_f_name, n, "%s/%s.pem", CERTS_DIR, hash); + if (access(tmp_f_name, F_OK) != -1) { + return tmp_f_name; + } + + snprintf(tmp_f_name, n, "%s/%s.pem", CA_ALL, hash); + if (access(tmp_f_name, F_OK) != -1) { + return tmp_f_name; + } + + // file not found + free(tmp_f_name); + return NULL; +} + +// take certificate hashes, check their validity and output json that +// will indicate which certificate were used for verification, whatever +// the chain was trusted and if all certificates needed for verification +// (with the exception of root CA) were present in hashes +int process_chain(const char **cert_hashes) +{ + int ret; + int rc; // return code from function + char *f_name; + + X509 *cert; + X509 *x509; + + X509_STORE *store; + + X509_STORE_CTX *csc; + + STACK_OF(X509) *ustack; + STACK_OF(X509) *vstack; + + // load certificates to temp structures + + // first the end entity cert + // (EE cert needs to be passed separately to OpenSSL verification context) + f_name = hash_to_filename(cert_hashes[0]); + if (f_name == NULL) + return 1; + + cert = load_cert(f_name); + free(f_name); + if (cert == NULL) { + printf("can't load certificate!\n"); + return 1; + } + + // then the intermediate certificates + ustack = sk_X509_new_null(); + + for (int i=1; cert_hashes[i]!=NULL; i++) { + //printf(".\n"); + f_name = hash_to_filename(cert_hashes[i]); + if (f_name == NULL) { + // file not found + continue; + } + x509 = load_cert(f_name); + if (x509 == NULL) { + // loading cert failed + continue; + } + sk_X509_push(ustack, x509); + free(f_name); + } + + // first try with just trusted certificates + + store = SSL_CTX_get_cert_store(trusted_only); + if (store == NULL) { + fprintf(stderr, "store init failed\n"); + return 1; + } + X509_STORE_set_flags(store, X509_V_FLAG_TRUSTED_FIRST); + + csc = X509_STORE_CTX_new(); + + ret = X509_STORE_CTX_init(csc, store, cert, ustack); + if (ret != 1) { + return 1; + } + + ret = X509_verify_cert(csc); + + if (ret != 1) { + // printf("%s\n", X509_verify_cert_error_string(csc->error)); + } else { + // chain is complete, output certificate hashes + printf("{\"chain\":\"complete\",\"certificates\":["); + vstack = X509_STORE_CTX_get_chain(csc); + for(int i=0; i= 0) { + lseek(fd, -1, SEEK_CUR); + } + + // parse the json object from the file + tok = json_tokener_new(); + do { + rc = read(fd, buffer, len); + if (rc < 0) + break; + obj = json_tokener_parse_ex(tok, buffer, rc); + } while ((jerr = json_tokener_get_error(tok)) == json_tokener_continue); + + if (jerr != json_tokener_success){ + fprintf(stderr, "error in file %s, line: %s\n", filename, buffer); + } + +tok_free: + + json_tokener_free(tok); + +close_fd: + close(fd); + +err: + if (ret) { + fprintf(stderr, "error while reading file: %i", ret); + } + return obj; +} + +// process all ciphersuites one by one from a given host results file +int process_host_results(char *filename) +{ + int fd; + int ret = 0; + int rc; + size_t sz; + size_t alloc_size = 64 * 1024; + const char *str; + struct json_object *root; + struct json_object *ciphers; + struct json_object *current; + struct json_object *certificates; + + struct json_object **known_chains; + known_chains = malloc(sizeof(struct json_object*) * 1); + known_chains[0] = NULL; + + struct lh_table *table; + enum json_type obj_t; + json_bool j_rc; + + root = read_json_from_file(filename); + if (root == NULL) { + ret = 1; + goto err; + } + + obj_t = json_object_get_type(root); + str = json_type_to_name(obj_t); + + j_rc = json_object_object_get_ex(root, "ciphersuite", &ciphers); + if (j_rc == FALSE) { + ret = 1; + goto json_free; + } + + // ok, we've got the ciphersuite part, we can print the json header for + // the host file + printf("{\"host\":\"%s\",\"chains\":[", filename); + + int first_printed=0; + for(int i=0; i < json_object_array_length(ciphers); i++) { + current = json_object_array_get_idx(ciphers, i); + //printf("\t[%i]:\n", i); + j_rc = json_object_object_get_ex(current, "certificates", &certificates); + if (j_rc == FALSE) + continue; + + const char** certs; + certs = calloc(sizeof(const char*), json_object_array_length(certificates) + 1); + int j; + for (j=0; j < json_object_array_length(certificates); j++) { + certs[j] = json_object_get_string(json_object_array_get_idx(certificates, j)); + //printf("\t\t\t%s\n", certs[j]); + } + rc = register_known_chains(&known_chains, certificates); + //printf("\t\t%i\n", rc); + + if (rc == 0 && j > 0) { + if (first_printed != 0) + printf(","); + if (process_chain(certs) != 0) { + fprintf(stderr, "error while processing chains!\n"); + } else { + first_printed = 1; + } + } + + // DEBUG, print whole json "object" object + //json_object_object_foreach(current, key, val) { + // str = json_object_to_json_string(val); + // printf("\t\t%s: %s\n", key, str); + //} + + free(certs); + } + printf("]}"); + +json_free: + json_object_put(root); + +err: + free(known_chains); + return ret; +} + +int main(int argc, char** argv) +{ + int ret; + + DIR *dirp; + struct dirent *direntp; + + char buffer[8192] = {}; + + SSL_load_error_strings(); + SSL_library_init(); + + /* init trust stores with certificate locations */ + trusted_only = SSL_CTX_new(SSLv23_method()); + if (trusted_only == NULL) { + ERR_print_errors_fp(stderr); + return 1; + } + + ret = SSL_CTX_load_verify_locations(trusted_only, NULL, CA_TRUSTED); + if (ret != 1) { + ERR_print_errors_fp(stderr); + return 1; + } + + all_CAs = SSL_CTX_new(SSLv23_method()); + if (all_CAs == NULL) { + ERR_print_errors_fp(stderr); + return 1; + } + + ret = SSL_CTX_load_verify_locations(all_CAs, NULL, CA_ALL); + if (ret != 1) { + ERR_print_errors_fp(stderr); + return 1; + } + + /* traverse the result directory, check all files in turn */ + dirp=opendir("results"); + while((direntp=readdir(dirp)) != NULL) { + if (strcmp(direntp->d_name, ".") == 0) + continue; + if (strcmp(direntp->d_name, "..") == 0) + continue; + + snprintf(buffer, 8191, "results/%s", direntp->d_name); + + ret = process_host_results(buffer); + if (ret == 1) { + fprintf(stderr, "error while processing %s\n", buffer); + } + if (ret == 0) + printf("\n"); + } + closedir(dirp); + + /* clean up */ + SSL_CTX_free(trusted_only); + SSL_CTX_free(all_CAs); + all_CAs = NULL; + trusted_only = NULL; + + return ret; +} From 9a956dc5a51c6991af25a8ec448d68a1ed958e52 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 3 Aug 2014 17:19:56 +0200 Subject: [PATCH 23/28] use pre-parsed data outputted by the C application --- top1m/parse_CAs.py | 239 +++++++++++---------------------------------- 1 file changed, 58 insertions(+), 181 deletions(-) diff --git a/top1m/parse_CAs.py b/top1m/parse_CAs.py index 72c2b8f..3b31183 100644 --- a/top1m/parse_CAs.py +++ b/top1m/parse_CAs.py @@ -18,37 +18,7 @@ import json import sys from collections import defaultdict import os -import re -import subprocess from OpenSSL import crypto -from M2Crypto import X509, EVP -from m2ext import _m2ext -from m2ext import SSL -import glob - -from pprint import pprint - -# override m2ext implementation so that it is possible to provide additional -# certificates to be used during verification. Requires m2ext with a small fix -# from https://github.com/tomato42/m2ext/tree/extended_ctx_init -class Context(SSL.Context): - def validate_certificate(self, cert, chain=None): - """ - Validate a certificate using this SSL Context - """ - if chain: - ptr = chain._ptr() - else: - ptr = None - store_ctx = X509.X509_Store_Context(_m2ext.x509_store_ctx_new(), _pyfree=1) - _m2ext.x509_store_ctx_init(store_ctx.ctx, - self.get_cert_store().store, - cert.x509, ptr) - rc = _m2ext.x509_verify_cert(store_ctx.ctx) - if rc < 0: - raise SSL.SSLError("Empty context") - return rc != 0 - invocations = defaultdict(int) @@ -86,98 +56,15 @@ def get_cert_subject_name(cert): else: return s_hash -def get_cert_hashes(path): - if path in subject_hashes: - return subject_hashes[path], issuer_hashes[path] - - with open(path) as srv_c_f: - srv_c_pem = srv_c_f.read() - - srv_c = crypto.load_certificate(crypto.FILETYPE_PEM, srv_c_pem) - - # can't make M2Crypto to output OpenSSL-compatible hashes... - subject_hash = ("%0.8X" % srv_c.get_subject().hash()).lower() - issuer_hash = ("%0.8X" % srv_c.get_issuer().hash()).lower() - - subject_hashes[path] = subject_hash - issuer_hashes[path] = issuer_hash - - return subject_hash, issuer_hash - -def gen_cert_paths(paths): - - # failsafe in case of a loop in path resolution - if len(paths) > 10: - return - - subject_hash, issuer_hash = get_cert_hashes(paths[-1]) - - if subject_hash == issuer_hash: - yield paths - else: - for ca_file in glob.glob(ca_certs_path + '/' + issuer_hash + ".*"): - for perm in gen_cert_paths(paths + [ca_file]): - if not perm in paths: - yield perm - -def is_chain_complete_f(file_names): - - stack = X509.X509_Stack() - for f_name in file_names[1:]: - cert = X509.load_cert(f_name) - stack.push(cert) - - cert = X509.load_cert(file_names[0]) - - return trusted_context.validate_certificate(cert, stack) - -def is_chain_complete(certs): - - stack = X509.X509_Stack() - - for cert in certs[1:]: - stack.push(cert) - - return trusted_context.validate_certificate(certs[0], stack) - -def is_chain_trusted(cert_hashes): - - c_hash = cert_hashes[0] - """ first check the likely option: the cert dir """ - file_name = certs_path + '/' + c_hash + '.pem' - if not os.path.exists(file_name): - """ then try the unlikely option: ca directory """ - file_name = ca_certs_path + '/' + c_hash + '.pem' - if not os.path.exists(file_name): - print "File with hash " + c_hash + " is missing!" - return False,None - - for cert_paths in gen_cert_paths([ file_name ]): - if is_chain_complete_f(cert_paths): - return True,cert_paths - - return False,None - def get_path_for_hash(cert_hash): - f_name = certs_path + '/' + c_hash + '.pem' + f_name = certs_path + '/' + cert_hash + '.pem' if not os.path.exists(f_name): - f_name = ca_certs_path + '/' + c_hash + '.pem' + f_name = ca_certs_path + '/' + cert_hash + '.pem' if not os.path.exists(f_name): #print "File with hash " + c_hash + " is missing!" return None return f_name -def is_chain_trusted_at_all(cert_list): - certs = [] - stack = X509.X509_Stack() - - cert = cert_list[0] - - for ca in cert_list[1:]: - stack.push(ca) - - return all_CAs_context.validate_certificate(cert, stack) - """ convert RSA and DSA key sizes to estimated Level of security """ def rsa_key_size_to_los(size): if size < 760: @@ -304,94 +191,84 @@ def collect_key_sizes(file_names): effective_security[security_level] += 1 -all_CAs_context = Context() -all_CAs_context.load_verify_locations(capath=ca_certs_path) -trusted_context = Context() -trusted_context.load_verify_locations(capath=trust_path) +with open("parsed") as res_file: + for line in res_file: + try: + res = json.loads(line) + except ValueError as e: + print "can't process line: " + line + continue -for r,d,flist in os.walk(path): - for f in flist: + f=res - server_chain_trusted = False - server_chain_complete = False - server_chains = [] - chains_tested = [] - valid = True + try: + server_chain_trusted = False + server_chain_complete = False + server_chains = [] + valid = False - """ process the file """ - f_abs = os.path.join(r,f) - with open(f_abs) as json_file: """ Keep certificates in memory for a given file """ known_certs = {} - """ discard files that fail to load """ - try: - results = json.load(json_file) - except ValueError: + if not "chains" in f: continue - """ discard files with empty results """ - if len(results['ciphersuite']) < 1: - continue + results = f["chains"] - valid = True + """ discard hosts with empty results """ + if len(results) < 1: + continue """ loop over list of ciphers """ - for entry in results['ciphersuite']: + for entry in results: - """ skip entries which don't have certificate references """ - if not 'certificates' in entry: + """ skip invalid results """ + if not 'chain' in entry: continue - """ skip entries for A(EC)DH suites """ - if len(entry['certificates']) < 1: + valid = True + + if entry['chain'] == "untrusted": continue - if not entry['certificates'] in chains_tested: - certs = [] + if entry['chain'] == "complete": + server_chain_complete = True + server_chain_trusted = True - for c_hash in entry['certificates']: - if c_hash in known_certs: - certs += [known_certs[c_hash]] - else: - path = get_path_for_hash(c_hash) - if path is None: - continue - cert = X509.load_cert(path) - known_certs[c_hash] = cert - certs += [cert] + if entry['chain'] == "incomplete": + server_chain_trusted = True - if is_chain_trusted_at_all(certs): - ret,tmpchain = is_chain_trusted(entry['certificates']) - if ret: - server_chain_trusted = True - if not tmpchain in server_chains: - server_chains += [tmpchain] - if is_chain_complete(certs): - server_chain_complete = True + server_chains += [entry['certificates']] - chains_tested += [entry['certificates']] - - if server_chain_trusted: - if server_chain_complete: - chains["complete"] += 1 - print "complete: " + f + if server_chain_trusted: + if server_chain_complete: + chains["complete"] += 1 + print "complete: " + f['host'] + else: + chains["incomplete"] += 1 + print "incomplete: " + f['host'] else: - chains["incomplete"] += 1 - print "incomplete: " + f - else: - chains["untrusted"] += 1 - print "untrusted: " + f + chains["untrusted"] += 1 + print "untrusted: " + f['host'] - if valid: - hosts += 1 + if valid: + hosts += 1 - for chain in server_chains: - collect_key_sizes(chain) - chain_len[str(len(chain))] += 1 - if len(chain) == 1: - print "file with chain 1 long " + f_abs - total += 1 + for chain in server_chains: + f_names = [] + for hash in chain: + path = get_path_for_hash(hash) + f_names += [path] + + collect_key_sizes(f_names) + chain_len[str(len(chain))] += 1 + if len(chain) == 1: + sys.stderr.write("file with chain 1 long: " + line) + total += 1 + except TypeError as e: + + sys.stderr.write("can't process: " + line) + continue """ Display stats """ #print "openssl invocations: " + str(invocations["openssl"]) From 3c93cbd6c234fe15f81ebc0243a252953c60aeb1 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 4 Aug 2014 17:22:53 +0200 Subject: [PATCH 24/28] make handling of self signed certs more robust openssl sometimes will print the filename, then the error, and finish with OK, matching the colon and space prevents from considering such certs to be valid --- cipherscan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index 693cf8a..db0178a 100755 --- a/cipherscan +++ b/cipherscan @@ -211,7 +211,7 @@ test_cipher_on_target() { local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ - grep 'OK$' >/dev/null; then + grep ': OK$' >/dev/null; then # if the certificate is an intermediate CA it may be useful # for connecting to servers that are misconfigured so save it From 1aeff568ee072be2427228218e0a7096d53a77ee Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 4 Oct 2014 12:51:34 +0200 Subject: [PATCH 25/28] update SEED and IDEA classification, do a total of broken ciphers SEED and IDEA are not good ciphers, but not broken, so count them separately, do a total count of servers that support broken and insecure ciphers --- top1m/parse_results.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 754cbb0..8e25419 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -147,6 +147,7 @@ for r,d,flist in os.walk(path): ciphertypes += 1 name = "z:" + entry['cipher'] tempcipherstats[name] = 1 + tempcipherstats['Insecure'] = 1 elif 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']: if not AESGCM: AESGCM = True @@ -171,10 +172,15 @@ for r,d,flist in os.walk(path): if not CHACHA20: ciphertypes += 1 CHACHA20 = True + elif 'IDEA' in entry['cipher'] or 'SEED' in entry['cipher']: + ciphertypes += 1 + name = "y:" + entry['cipher'] + tempcipherstats[name] = 1 else: ciphertypes += 1 name = "z:" + entry['cipher'] tempcipherstats[name] = 1 + tempcipherstats['Insecure'] = 1 """ store key handshake methods """ if 'ECDHE' in entry['cipher']: From 0adf721643ad8eac02f0de18b889db654056b896 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 4 Oct 2014 13:27:48 +0200 Subject: [PATCH 26/28] parse_CAs.py - add support for MD5 sigs --- top1m/parse_CAs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/top1m/parse_CAs.py b/top1m/parse_CAs.py index 3b31183..d564c28 100644 --- a/top1m/parse_CAs.py +++ b/top1m/parse_CAs.py @@ -85,7 +85,9 @@ def rsa_key_size_to_los(size): return 256 def sig_alg_to_los(name): - if 'SHA1' in name.upper(): + if 'MD5' in name.upper(): + return 64 + elif 'SHA1' in name.upper(): return 80 elif 'SHA224' in name.upper(): return 112 From 8911827be146646cee6297f558c572564c4a7f77 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 4 Oct 2014 13:41:40 +0200 Subject: [PATCH 27/28] process-certificate-statistics.sh - the script HOWTO to turn results to CA stats --- top1m/process-certificate-statistics.sh | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 top1m/process-certificate-statistics.sh diff --git a/top1m/process-certificate-statistics.sh b/top1m/process-certificate-statistics.sh new file mode 100755 index 0000000..b9256d8 --- /dev/null +++ b/top1m/process-certificate-statistics.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +if [ ! -d ./ca_files ]; then + echo "Directory with collected CA certificates missing!" >&2 + exit 1 +fi + +if [ ! -d ./ca_trusted ]; then + echo "Directory with just trust anchors missing!" >&2 + exit 1 +fi + +if [ ! -d ./certs ]; then + echo "Directory with certificates missing!" >&2 + exit 1 +fi + +if ! ls -f ./ca_files/????????.? > /dev/null; then + echo "CA certificates directory not hashed properly (use c_rehash)" >&2 + exit 1 +fi + +if ! ls -f ./ca_trusted/????????.? > /dev/null; then + echo "Directory with trust anchors not hashed properly (use c_rehash)" >&2 + exit 1 +fi + +if [ ! -d ./results ]; then + echo "Directory with scan results missing!" >&2 + exit 1 +fi + +if [ ! -x ./parse_CAs ]; then + echo "Compiling parse_CAs script" + gcc -o parse_CAs parse_CAs.c -lssl -lcrypto -ljson-c --std=gnu99 + if [ $? -ne 0 ]; then + echo "Compilation failed, aborting" >&2 + exit 1 + fi +fi + +echo "Verifying certificate chains from results files" +./parse_CAs > parsed +echo "Calculating statistics for verified certificate chains" +python parse_CAs.py > trust_scan +echo "Done!" +echo "Results are in \"trust_scan\" file" From 77f326522e253ba0b3c0cc8ac7cb4f43d884f3be Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 4 Oct 2014 14:46:36 +0200 Subject: [PATCH 28/28] fixes for the pull request #18 there were few small issues with the pull #18 even though jvehent merged it, this fixes them --- cipherscan | 2 ++ top1m/parse_results.py | 1 + top1m/testtop1m.sh | 8 +++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cipherscan b/cipherscan index db0178a..2f3b024 100755 --- a/cipherscan +++ b/cipherscan @@ -22,6 +22,8 @@ fi if [ ! -e "$CACERTS" ]; then echo "Warning: CA Certificates not found at $CACERTS, export CACERTS variable with location of your trust anchors" 1>&2 fi +# RSA ciphers are put at the end to force Google servers to accept ECDSA ciphers +# (probably a result of a workaround for the bug in Apple implementation of ECDSA) CIPHERSUITE="ALL:COMPLEMENTOFALL:+aRSA" DEBUG=0 VERBOSE=0 diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 8e25419..0ac7c4f 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -75,6 +75,7 @@ for r,d,flist in os.walk(path): tempdsakeystats = {} tempsigstats = {} tempticketstats = {} + """ supported ciphers by the server under scan """ tempcipherstats = {} ciphertypes = 0 AESGCM = False diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index 0f07744..6021a3e 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -2,7 +2,7 @@ parallel=10 max_bg=50 absolute_max_bg=100 -max_load=50 +max_load_avg=50 if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then echo "max user processes too low, use ulimit -u to increase" @@ -38,8 +38,8 @@ function wait_for_jobs() { local no_jobs no_jobs=$(jobs | wc -l) - while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do - if awk -v maxload=$max_load '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then + while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load_avg '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do + if awk -v maxload=$max_load_avg '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then return fi sleep 1 @@ -72,6 +72,8 @@ function scan_host_no_sni() { } function scan_hostname() { + # check if the hostname isn't an IP address (since we can't put IP + # addresses to SNI extension) if [[ ! -z $(awk -F. '$1>=0 && $1<=255 && $2>=0 && $2<=255 && $3>=0 && $3<=255 && $4>=0 && $4<=255 && NF==4' <<<"$1") ]]; then scan_host_no_sni $1