Merge pull request #15 from tomato42/reporting-improvements-03

Reporting improvements 03

top1m/parse_results.py

@@ -9,10 +9,15 @@ import sys
 from collections import defaultdict
 import os
 
+report_untrused=False
+
 cipherstats = defaultdict(int)
 pfsstats = defaultdict(int)
 protocolstats = defaultdict(int)
 handshakestats = defaultdict(int)
+keysize = defaultdict(int)
+sigalg = defaultdict(int)
+dsarsastack = 0
 total = 0
 for r,d,flist in os.walk(path):
 
@@ -20,9 +25,14 @@ for r,d,flist in os.walk(path):
 
         """ initialize variables for stats of the current site """
         temppfsstats = {}
+        tempkeystats = {}
+        tempecckeystats = {}
+        tempdsakeystats = {}
+        tempsigstats = {}
         ciphertypes = 0
         AESGCM = False
         AES = False
+        CHACHA20 = False
         DES3 = False
         CAMELLIA = False
         RC4 = False
@@ -34,6 +44,9 @@ for r,d,flist in os.walk(path):
         TLS1 = False
         TLS1_1 = False
         TLS1_2 = False
+        dualstack = False
+        ECDSA = False
+        trusted = False
 
         """ process the file """
         f_abs = os.path.join(r,f)
@@ -48,13 +61,17 @@ for r,d,flist in os.walk(path):
             if len(results['ciphersuite']) < 1:
                 continue
 
-            total += 1
-
             """ loop over list of ciphers """
             for entry in results['ciphersuite']:
 
+                # some servers return different certificates with different
+                # ciphers, also we may become redirected to other server with
+                # different config (because over-reactive IPS)
+                if 'False' in entry['trusted'] and report_untrused == False:
+                    continue
+
                 """ store the ciphers supported """
-                if 'AES-GCM' in entry['cipher']:
+                if 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']:
                     if not AESGCM:
                         AESGCM = True
                         ciphertypes += 1
@@ -74,6 +91,10 @@ for r,d,flist in os.walk(path):
                     if not RC4:
                         ciphertypes += 1
                         RC4 = True
+                elif 'CHACHA20' in entry['cipher']:
+                    if not CHACHA20:
+                        ciphertypes += 1
+                        CHACHA20 = True
                 else:
                     ciphertypes += 1
                     name = "z:" + entry['cipher']
@@ -87,6 +108,25 @@ for r,d,flist in os.walk(path):
                     DHE = True
                     temppfsstats[entry['pfs']] = 1
 
+                """ save the key size """
+                if 'ECDSA' in entry['cipher']:
+                    ECDSA = True
+                    tempecckeystats[entry['pubkey'][0]] = 1
+                elif 'DSS' in entry['cipher']:
+                    tempdsakeystats[entry['pubkey'][0]] = 1
+                elif 'AECDH' in entry['cipher'] or 'ADH' in entry['cipher']:
+                    """ skip """
+                else:
+                    tempkeystats[entry['pubkey'][0]] = 1
+                    if ECDSA:
+                        dualstack = True
+
+                if 'True' in entry['trusted'] and not 'ADH' in entry['cipher'] and not 'AECDH' in entry['cipher']:
+                    trusted = True
+
+                """ save key signatures size """
+                tempsigstats[entry['sigalg'][0]] = 1
+
                 """ store the versions of TLS supported """
                 for protocol in entry['protocols']:
                     if protocol == 'SSLv2':
@@ -101,14 +141,34 @@ for r,d,flist in os.walk(path):
                         TLS1_2 = True
         json_file.close()
 
+        """ don't store stats from unusued servers """
+        if report_untrused == False and trusted == False:
+            continue
+
+        total += 1
+
         """ done with this file, storing the stats """
         if DHE or ECDHE:
             pfsstats['Support PFS'] += 1
             if 'DHE-' in results['ciphersuite'][0]['cipher']:
                 pfsstats['Prefer PFS'] += 1
+                pfsstats['Prefer ' + results['ciphersuite'][0]['pfs']] += 1
             for s in temppfsstats:
                 pfsstats[s] += 1
 
+        for s in tempkeystats:
+            keysize['RSA ' + s] += 1
+        for s in tempecckeystats:
+            keysize['ECDSA ' + s] += 1
+        for s in tempdsakeystats:
+            keysize['DSA ' + s] += 1
+
+        if dualstack:
+            dsarsastack += 1
+
+        for s in tempsigstats:
+            sigalg[s] += 1
+
         """ store cipher stats """
         if AESGCM:
             cipherstats['AES-GCM'] += 1
@@ -117,7 +177,14 @@ for r,d,flist in os.walk(path):
         if AES:
             cipherstats['AES'] += 1
             if ciphertypes == 1:
+                cipherstats['AES-CBC Only'] += 1
+        if (AES and ciphertypes == 1) or (AESGCM and ciphertypes == 1)\
+            or (AES and AESGCM and ciphertypes == 2):
                 cipherstats['AES Only'] += 1
+        if CHACHA20:
+            cipherstats['CHACHA20'] += 1
+            if ciphertypes == 1:
+                cipherstats['CHACHA20 Only'] += 1
         if DES3:
             cipherstats['3DES'] += 1
             if ciphertypes == 1:
@@ -130,6 +197,12 @@ for r,d,flist in os.walk(path):
             cipherstats['RC4'] += 1
             if ciphertypes == 1:
                 cipherstats['RC4 Only'] += 1
+            if 'RC4' in results['ciphersuite'][0]['cipher']:
+                if 'TLSv1.1' in results['ciphersuite'][0]['protocols'] or\
+                   'TLSv1.2' in results['ciphersuite'][0]['protocols']:
+                        cipherstats['RC4 forced in TLS1.1+'] += 1
+                cipherstats['RC4 Preferred'] += 1
+
 
         """ store handshake stats """
         if ECDHE:
@@ -152,22 +225,30 @@ for r,d,flist in os.walk(path):
             protocolstats['TLS1'] += 1
             if not SSL2 and not SSL3 and not TLS1_1 and not TLS1_2:
                 protocolstats['TLS1 Only'] += 1
+        if not SSL2 and (SSL3 or TLS1) and not TLS1_1 and not TLS1_2:
+            protocolstats['SSL3 or TLS1 Only'] += 1
+        if not SSL2 and not SSL3 and not TLS1:
+            protocolstats['TLS1.1 or up Only'] += 1
         if TLS1_1:
             protocolstats['TLS1.1'] += 1
             if not SSL2 and not SSL3 and not TLS1 and not TLS1_2:
-                protocolstats['TLS1_1 Only'] += 1
+                protocolstats['TLS1.1 Only'] += 1
         if TLS1_2:
             protocolstats['TLS1.2'] += 1
             if not SSL2 and not SSL3 and not TLS1 and not TLS1_1:
                 protocolstats['TLS1.2 Only'] += 1
-        if TLS1_2 and not TLS1_1:
+        if TLS1_2 and not TLS1_1 and TLS1:
-            protocolstats['TLS1.2 but not 1.1'] += 1
+            protocolstats['TLS1.2, 1.0 but not 1.1'] += 1
 
     # for testing, break early
     #if total % 1999 == 0:
     #    break
 
 print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total)
+if report_untrused == False:
+    print("Stats only from connections that did provide valid certificates")
+    print("(or anonymous DH from servers that do also have valid certificate installed)\n")
+
 """ Display stats """
 print("\nSupported Ciphers         Count     Percent")
 print("-------------------------+---------+-------")
@@ -192,6 +273,20 @@ for stat in sorted(pfsstats):
         pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4)
     sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n")
 
+print("\nCertificate sig alg     Count     Percent ")
+print("-------------------------+---------+--------")
+for stat in sorted(sigalg):
+    percent = round(sigalg[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(sigalg[stat]).ljust(10) + str(percent).ljust(9) + "\n")
+
+print("\nCertificate key size    Count     Percent ")
+print("-------------------------+---------+--------")
+for stat in sorted(keysize):
+    percent = round(keysize[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(9) + "\n")
+
+sys.stdout.write("RSA/ECDSA Dual Stack".ljust(25) + " " + str(dsarsastack).ljust(10) + str(round(dsarsastack/total * 100, 4)) + "\n")
+
 print("\nSupported Protocols       Count     Percent")
 print("-------------------------+---------+-------")
 for stat in sorted(protocolstats):