From 7a92186122e83f556487331d384a5811dc7cd45a Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 4 Apr 2014 20:12:50 +0200 Subject: [PATCH 01/12] Improve scanning performance and reduce false negatives scan all the machines from top-1m.csv file, wait for completion of all jobs i=1 is an off-by-one-error support top-1m.csv files with arbitrary number of sites run scans for many hosts at a time, but don't run more than specified amount in case where default domain name doesn't resolve or doesn't have port 443 open, retry with www. prefix --- top1m/testtop1m.sh | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index 52f504b..71b1b0f 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -1,14 +1,37 @@ #!/usr/bin/env bash parallel=50 +max_bg=400 [ ! -e "results" ] && mkdir results -i=1 -while [ $i -lt 1000000 ] + +function wait_for_jobs() { + local no_jobs + no_jobs=$(jobs | wc -l) + + while [ $no_jobs -gt $1 ]; do + sleep 1 + no_jobs=$(jobs | wc -l) + done +} + +i=0 +count=$(wc -l top-1m.csv | awk '{print $1}') +while [ $i -lt $count ] do echo processings sites $i to $((i + parallel)) - for t in $(tail -$((1000000 - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2) + for t in $(tail -$(($count - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2) do - (tcping -u 10000000 $t 443; if [ $? -gt 0 ];then continue;fi;../cipherscan $t:443 -json > results/$t )& + (tcping -u 10000000 $t 443; + if [ $? -gt 0 ];then + tcping -u 10000000 www.$t 443; + if [ $? -gt 0 ]; then + continue; + else + ../cipherscan -json www.$t:443 > results/www.$t + continue; + fi; + fi;../cipherscan -json $t:443 > results/$t )& done - sleep 7 i=$(( i + parallel)) + wait_for_jobs $max_bg done +wait From 8b2b6f591698d8e6984d29aee96a73fb272f1f43 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 4 Apr 2014 20:18:36 +0200 Subject: [PATCH 02/12] parsing of signature algorithm and key size add parsing of signature algorithm and key size from the individual results, report summary --- top1m/parse_results.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 90306d2..6fd52bd 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -13,6 +13,8 @@ cipherstats = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) handshakestats = defaultdict(int) +keysize = defaultdict(int) +sigalg = defaultdict(int) total = 0 for r,d,flist in os.walk(path): @@ -20,6 +22,10 @@ for r,d,flist in os.walk(path): """ initialize variables for stats of the current site """ temppfsstats = {} + tempkeystats = {} + tempecckeystats = {} + tempdsakeystats = {} + tempsigstats = {} ciphertypes = 0 AESGCM = False AES = False @@ -87,6 +93,17 @@ for r,d,flist in os.walk(path): DHE = True temppfsstats[entry['pfs']] = 1 + """ save the key size """ + if 'ECDSA' in entry['cipher']: + tempecckeystats[entry['pubkey'][0]] = 1 + elif 'DSS' in entry['cipher']: + tempdsakeystats[entry['pubkey'][0]] = 1 + else: + tempkeystats[entry['pubkey'][0]] = 1 + + """ save key signatures size """ + tempsigstats[entry['sigalg'][0]] = 1 + """ store the versions of TLS supported """ for protocol in entry['protocols']: if protocol == 'SSLv2': @@ -109,6 +126,16 @@ for r,d,flist in os.walk(path): for s in temppfsstats: pfsstats[s] += 1 + for s in tempkeystats: + keysize['RSA ' + s] += 1 + for s in tempecckeystats: + keysize['ECDSA ' + s] += 1 + for s in tempdsakeystats: + keysize['DSA ' + s] += 1 + + for s in tempsigstats: + sigalg[s] += 1 + """ store cipher stats """ if AESGCM: cipherstats['AES-GCM'] += 1 @@ -192,6 +219,18 @@ for stat in sorted(pfsstats): pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n") +print("\nCertificate sig alg Count Percent ") +print("-------------------------+---------+--------") +for stat in sorted(sigalg): + percent = round(sigalg[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(sigalg[stat]).ljust(10) + str(percent).ljust(9) + "\n") + +print("\nCertificate key size Count Percent ") +print("-------------------------+---------+--------") +for stat in sorted(keysize): + percent = round(keysize[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(9) + "\n") + print("\nSupported Protocols Count Percent") print("-------------------------+---------+-------") for stat in sorted(protocolstats): From 167ef8b50277f2b33a813d36e4d366bef1d7509a Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 4 Apr 2014 21:08:38 +0200 Subject: [PATCH 03/12] report number of servers that use ECDSA and RSA certificates Since use of both ECDSA and RSA certificates is easy, it is relatively simple to support both. Report the total number of such servers --- top1m/parse_results.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 6fd52bd..0a9817b 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -15,6 +15,7 @@ protocolstats = defaultdict(int) handshakestats = defaultdict(int) keysize = defaultdict(int) sigalg = defaultdict(int) +dsarsastack = 0 total = 0 for r,d,flist in os.walk(path): @@ -40,6 +41,8 @@ for r,d,flist in os.walk(path): TLS1 = False TLS1_1 = False TLS1_2 = False + dualstack = False + ECDSA = False """ process the file """ f_abs = os.path.join(r,f) @@ -95,11 +98,16 @@ for r,d,flist in os.walk(path): """ save the key size """ if 'ECDSA' in entry['cipher']: + ECDSA = True tempecckeystats[entry['pubkey'][0]] = 1 elif 'DSS' in entry['cipher']: tempdsakeystats[entry['pubkey'][0]] = 1 + elif 'AECDH' in entry['cipher'] or 'ADH' in entry['cipher']: + """ skip """ else: tempkeystats[entry['pubkey'][0]] = 1 + if ECDSA: + dualstack = True """ save key signatures size """ tempsigstats[entry['sigalg'][0]] = 1 @@ -133,6 +141,9 @@ for r,d,flist in os.walk(path): for s in tempdsakeystats: keysize['DSA ' + s] += 1 + if dualstack: + dsarsastack += 1 + for s in tempsigstats: sigalg[s] += 1 @@ -231,6 +242,8 @@ for stat in sorted(keysize): percent = round(keysize[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(9) + "\n") +sys.stdout.write("RSA/ECDSA Dual Stack".ljust(25) + " " + str(dsarsastack).ljust(10) + str(round(dsarsastack/total * 100, 4)) + "\n") + print("\nSupported Protocols Count Percent") print("-------------------------+---------+-------") for stat in sorted(protocolstats): From 5d7ec3a714a8c3ade2c9a85046d4d85c55a6addb Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 5 Apr 2014 01:33:33 +0200 Subject: [PATCH 04/12] add ability to ignore results from untrusted servers --- top1m/parse_results.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 0a9817b..51b84c1 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -9,6 +9,8 @@ import sys from collections import defaultdict import os +report_untrused=False + cipherstats = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) @@ -43,6 +45,7 @@ for r,d,flist in os.walk(path): TLS1_2 = False dualstack = False ECDSA = False + trusted = False """ process the file """ f_abs = os.path.join(r,f) @@ -57,11 +60,12 @@ for r,d,flist in os.walk(path): if len(results['ciphersuite']) < 1: continue - total += 1 - """ loop over list of ciphers """ for entry in results['ciphersuite']: + if 'True' in entry['trusted']: + trusted = True + """ store the ciphers supported """ if 'AES-GCM' in entry['cipher']: if not AESGCM: @@ -126,6 +130,12 @@ for r,d,flist in os.walk(path): TLS1_2 = True json_file.close() + """ don't store stats from unusued servers """ + if report_untrused == False and trusted == False: + continue + + total += 1 + """ done with this file, storing the stats """ if DHE or ECDHE: pfsstats['Support PFS'] += 1 From bc0409ca735164340fdd0f563729d32e8b00c9d9 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 5 Apr 2014 20:21:35 +0200 Subject: [PATCH 05/12] in "no-untrusted mode": filter out ADH and AECDH suites If server negotiates ADH or AECDH suite, openssl returns "ok" in cert checking. Don't mark server as trusted because of that. Don't collect statistics on servers that provide only untrusted connections. --- top1m/parse_results.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 51b84c1..70cf39a 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -63,8 +63,11 @@ for r,d,flist in os.walk(path): """ loop over list of ciphers """ for entry in results['ciphersuite']: - if 'True' in entry['trusted']: - trusted = True + # some servers return different certificates with different + # ciphers, also we may become redirected to other server with + # different config (because over-reactive IPS) + if 'False' in entry['trusted'] and report_untrused == False: + continue; """ store the ciphers supported """ if 'AES-GCM' in entry['cipher']: @@ -113,6 +116,9 @@ for r,d,flist in os.walk(path): if ECDSA: dualstack = True + if 'True' in entry['trusted'] and not 'ADH' in entry['cipher'] and not 'AECDH' in entry['cipher']: + trusted = True + """ save key signatures size """ tempsigstats[entry['sigalg'][0]] = 1 @@ -216,6 +222,10 @@ for r,d,flist in os.walk(path): # break print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total) +if report_untrused == False: + print("Stats only from connections that did provide valid certificates") + print("(or anonymous DH from servers that do also have valid certificate installed)\n") + """ Display stats """ print("\nSupported Ciphers Count Percent") print("-------------------------+---------+-------") From fedf12dd5a3e04f76b7a2e3794f629483a2c4f1a Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Apr 2014 14:06:18 +0200 Subject: [PATCH 06/12] fix spelling in TLS stats (TLS1_1 vs TLS1.1) --- top1m/parse_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 70cf39a..7e5b1ee 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -209,7 +209,7 @@ for r,d,flist in os.walk(path): if TLS1_1: protocolstats['TLS1.1'] += 1 if not SSL2 and not SSL3 and not TLS1 and not TLS1_2: - protocolstats['TLS1_1 Only'] += 1 + protocolstats['TLS1.1 Only'] += 1 if TLS1_2: protocolstats['TLS1.2'] += 1 if not SSL2 and not SSL3 and not TLS1 and not TLS1_1: From 1536064528787ff145df512e2a953d1ad7951023 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Apr 2014 14:09:03 +0200 Subject: [PATCH 07/12] fix and extend reporting of AES-GCM ciphers AES-GCM ciphers don't have "AES-GCM" substring in the openssl name extend reporting of AES ciphers, split to AES-CBC, AES-GCM and AES in general --- top1m/parse_results.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 7e5b1ee..d9029fb 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -70,7 +70,7 @@ for r,d,flist in os.walk(path): continue; """ store the ciphers supported """ - if 'AES-GCM' in entry['cipher']: + if 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']: if not AESGCM: AESGCM = True ciphertypes += 1 @@ -171,6 +171,9 @@ for r,d,flist in os.walk(path): if AES: cipherstats['AES'] += 1 if ciphertypes == 1: + cipherstats['AES-CBC Only'] += 1 + if (AES and ciphertypes == 1) or (AESGCM and ciphertypes == 1)\ + or (AES and AESGCM and ciphertypes == 2): cipherstats['AES Only'] += 1 if DES3: cipherstats['3DES'] += 1 From b2edb0e36141c5d7770678e2fc098b52696ca47c Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Apr 2014 14:11:52 +0200 Subject: [PATCH 08/12] add support for Chacha20 based ciphers Basically all Google servers support Chacha20 now and it is not a bad choice, so report it as a regular cipher --- top1m/parse_results.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index d9029fb..2f6adc9 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -32,6 +32,7 @@ for r,d,flist in os.walk(path): ciphertypes = 0 AESGCM = False AES = False + CHACHA20 = False DES3 = False CAMELLIA = False RC4 = False @@ -90,6 +91,10 @@ for r,d,flist in os.walk(path): if not RC4: ciphertypes += 1 RC4 = True + elif 'CHACHA20' in entry['cipher']: + if not CHACHA20: + ciphertypes += 1 + CHACHA20 = True else: ciphertypes += 1 name = "z:" + entry['cipher'] @@ -175,6 +180,10 @@ for r,d,flist in os.walk(path): if (AES and ciphertypes == 1) or (AESGCM and ciphertypes == 1)\ or (AES and AESGCM and ciphertypes == 2): cipherstats['AES Only'] += 1 + if CHACHA20: + cipherstats['CHACHA20'] += 1 + if ciphertypes == 1: + cipherstats['CHACHA20 Only'] += 1 if DES3: cipherstats['3DES'] += 1 if ciphertypes == 1: From 15d77628526c9199a889182c5669cad64b50e186 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Apr 2014 14:13:44 +0200 Subject: [PATCH 09/12] fix reporting of the TLS1.2 but not TLS1.1 Some servers may be configured to support only TLS1.2, it would count them towards the number of servers affected by the OpenSSL bug --- top1m/parse_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 2f6adc9..3f58aac 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -226,8 +226,8 @@ for r,d,flist in os.walk(path): protocolstats['TLS1.2'] += 1 if not SSL2 and not SSL3 and not TLS1 and not TLS1_1: protocolstats['TLS1.2 Only'] += 1 - if TLS1_2 and not TLS1_1: - protocolstats['TLS1.2 but not 1.1'] += 1 + if TLS1_2 and not TLS1_1 and TLS1: + protocolstats['TLS1.2, 1.0 but not 1.1'] += 1 # for testing, break early #if total % 1999 == 0: From 39c3e1978a65f69348e714a5cec15c8717b54bfc Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Apr 2014 14:15:32 +0200 Subject: [PATCH 10/12] more detailed PFS report Just because server supports some bad DH params, doesn't mean it will force them on users. Report number of servers that prefer specific DH params. --- top1m/parse_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 3f58aac..06c4152 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -152,6 +152,7 @@ for r,d,flist in os.walk(path): pfsstats['Support PFS'] += 1 if 'DHE-' in results['ciphersuite'][0]['cipher']: pfsstats['Prefer PFS'] += 1 + pfsstats['Prefer ' + results['ciphersuite'][0]['pfs']] += 1 for s in temppfsstats: pfsstats[s] += 1 From 010ebccd806675eeb4d24a8f3ebaf9192048f9de Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Apr 2014 14:17:24 +0200 Subject: [PATCH 11/12] extend SSL stats Two interesting server configurations are ones that support only SSL3 or TLS1 only (old, but otherwise correctly configured servers) and ones that support only TLS1.1 or up (brave admins that support only new clients) --- top1m/parse_results.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 06c4152..66eede9 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -219,6 +219,10 @@ for r,d,flist in os.walk(path): protocolstats['TLS1'] += 1 if not SSL2 and not SSL3 and not TLS1_1 and not TLS1_2: protocolstats['TLS1 Only'] += 1 + if not SSL2 and (SSL3 or TLS1) and not TLS1_1 and not TLS1_2: + protocolstats['SSL3 or TLS1 Only'] += 1 + if not SSL2 and not SSL3 and not TLS1: + protocolstats['TLS1.1 or up Only'] += 1 if TLS1_1: protocolstats['TLS1.1'] += 1 if not SSL2 and not SSL3 and not TLS1 and not TLS1_2: From 0acd31af53780aadcd8b0f87bfd67c36c02dfad7 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Apr 2014 14:19:37 +0200 Subject: [PATCH 12/12] extend reporting of RC4-related stats While preferring RC4 in TLS1.0 or SSL3 was recommended before, it was always known that TLS1.1 and TLS1.2 were not vulnerable against BEAST, so forcing RC4 there is a mistake. Report number of such servers. --- top1m/parse_results.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 66eede9..9509510 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -197,6 +197,12 @@ for r,d,flist in os.walk(path): cipherstats['RC4'] += 1 if ciphertypes == 1: cipherstats['RC4 Only'] += 1 + if 'RC4' in results['ciphersuite'][0]['cipher']: + if 'TLSv1.1' in results['ciphersuite'][0]['protocols'] or\ + 'TLSv1.2' in results['ciphersuite'][0]['protocols']: + cipherstats['RC4 forced in TLS1.1+'] += 1 + cipherstats['RC4 Preferred'] += 1 + """ store handshake stats """ if ECDHE: