add collection of LoS for the cert paths found

top1m/parse_CAs.py

@@ -57,9 +57,11 @@ hosts = 0
 chains = defaultdict(int)
 chain_len = defaultdict(int)
 keysize = defaultdict(int)
+keysize_per_chain = defaultdict(int)
 root_CA = defaultdict(int)
 sig_alg = defaultdict(int)
 intermediate_CA = defaultdict(int)
+effective_security = defaultdict(int)
 
 subject_hashes = {}
 issuer_hashes = {}
@@ -161,7 +163,7 @@ def get_path_for_hash(cert_hash):
     if not os.path.exists(f_name):
         f_name = ca_certs_path + '/' + c_hash + '.pem'
         if not os.path.exists(f_name):
-            print "File with hash " + c_hash + " is missing!"
+            #print "File with hash " + c_hash + " is missing!"
             return None
     return f_name
 
@@ -176,8 +178,43 @@ def is_chain_trusted_at_all(cert_list):
 
     return all_CAs_context.validate_certificate(cert, stack)
 
+""" convert RSA and DSA key sizes to estimated Level of security """
+def rsa_key_size_to_los(size):
+    if size < 760:
+        return 40
+    elif size < 1020:
+        return 64
+    elif size < 2040:
+        return 80
+    elif size < 3068:
+        return 112
+    elif size < 4094:
+        return 128
+    elif size < 7660:
+        return 152
+    elif size < 15300:
+        return 192
+    else:
+        return 256
+
+def sig_alg_to_los(name):
+    if 'SHA1' in name.upper():
+        return 80
+    elif 'SHA224' in name.upper():
+        return 112
+    elif 'SHA256' in name.upper():
+        return 128
+    elif 'SHA384' in name.upper():
+        return 192
+    elif 'SHA512' in name.upper():
+        return 256
+    else:
+        raise UnknownSigAlgError
+
 def collect_key_sizes(file_names):
 
+    tmp_keysize = {}
+
     """ don't collect signature alg for the self signed root """
     with open(file_names[-1]) as cert_file:
         cert_pem = cert_file.read()
@@ -187,12 +224,19 @@ def collect_key_sizes(file_names):
     pubkey = cert.get_pubkey()
     if pubkey.type() == crypto.TYPE_RSA:
         keysize['RSA ' + str(pubkey.bits())] += 1
+        tmp_keysize['RSA ' + str(pubkey.bits())] = 1
+        security_level = rsa_key_size_to_los(pubkey.bits())
     elif pubkey.type() == crypto.TYPE_DSA:
         keysize['DSA ' + str(pubkey.bits())] += 1
+        tmp_keysize['DSA ' + str(pubkey.bits())] = 1
+        security_level = rsa_key_size_to_los(pubkey.bits())
     elif pubkey.type() == 408:
         keysize['ECDSA ' + str(pubkey.bits())] += 1
+        tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1
+        security_level = pubkey.bits()/2
     else:
         keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1
+        security_level = 0
 
     root_CA[get_cert_subject_name(cert)] += 1
 
@@ -206,17 +250,60 @@ def collect_key_sizes(file_names):
         pubkey = cert.get_pubkey()
         if pubkey.type() == crypto.TYPE_RSA:
             keysize['RSA ' + str(pubkey.bits())] += 1
+            tmp_keysize['RSA ' + str(pubkey.bits())] = 1
+            c_key_level = rsa_key_size_to_los(pubkey.bits())
         elif pubkey.type() == crypto.TYPE_DSA:
             keysize['DSA ' + str(pubkey.bits())] += 1
+            tmp_keysize['DSA ' + str(pubkey.bits())] = 1
+            c_key_level = rsa_key_size_to_los(pubkey.bits())
         elif pubkey.type() == 408:
             keysize['ECDSA ' + str(pubkey.bits())] += 1
+            tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1
+            c_key_level = pubkey.bits() / 2
         else:
             keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1
+            c_key_level = 0
+
+        if security_level > c_key_level:
+            security_level = c_key_level
 
         sig_alg[cert.get_signature_algorithm()] += 1
+        c_sig_level = sig_alg_to_los(cert.get_signature_algorithm())
+        if security_level > c_sig_level:
+            security_level = c_sig_level
 
         intermediate_CA[get_cert_subject_name(cert)] += 1
 
+    for key_s in tmp_keysize:
+        keysize_per_chain[key_s] += 1
+
+    # XXX doesn't handle the situation in which the CA uses its certificate
+    # for a web server properly
+    with open(file_names[0]) as cert_file:
+       cert_pem = cert_file.read()
+
+    cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+
+    pubkey = cert.get_pubkey()
+    if pubkey.type() == crypto.TYPE_RSA:
+       c_key_level = rsa_key_size_to_los(pubkey.bits())
+    elif pubkey.type() == crypto.TYPE_DSA:
+       c_key_level = rsa_key_size_to_los(pubkey.bits())
+    elif pubkey.type() == 408:
+       c_key_level = pubkey.bits() / 2
+    else:
+       c_key_level = 0
+
+    if security_level > c_key_level:
+       security_level = c_key_level
+
+    c_sig_level = sig_alg_to_los(cert.get_signature_algorithm())
+    if security_level > c_sig_level:
+       security_level = c_sig_level
+
+    effective_security[security_level] += 1
+
+
 all_CAs_context = Context()
 all_CAs_context.load_verify_locations(capath=ca_certs_path)
 trusted_context = Context()
@@ -267,7 +354,10 @@ for r,d,flist in os.walk(path):
                         if c_hash in known_certs:
                             certs += [known_certs[c_hash]]
                         else:
-                            cert = X509.load_cert(get_path_for_hash(c_hash))
+                            path = get_path_for_hash(c_hash)
+                            if path is None:
+                                continue
+                            cert = X509.load_cert(path)
                             known_certs[c_hash] = cert
                             certs += [cert]
 
@@ -285,10 +375,13 @@ for r,d,flist in os.walk(path):
         if server_chain_trusted:
             if server_chain_complete:
                 chains["complete"] += 1
+                print "complete: " + f
             else:
                 chains["incomplete"] += 1
+                print "incomplete: " + f
         else:
             chains["untrusted"] += 1
+            print "untrusted: " + f
 
         if valid:
             hosts += 1
@@ -321,11 +414,27 @@ for stat in sorted(chain_len):
     percent = round(chain_len[stat] / total * 100, 4)
     sys.stdout.write(stat.ljust(25) + " " + str(chain_len[stat]).ljust(10) + str(percent).ljust(4) + "\n")
 
-print("\nCA Key Size               Count     Percent")
+print("\nCA key size in chains     Count")
-print("-------------------------+---------+-------")
+print("-------------------------+---------")
 for stat in sorted(keysize):
-    percent = round(keysize[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + "\n")
-    sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nChains with CA key        Count     Percent")
+print("-------------------------+---------+-------")
+for stat in sorted(keysize_per_chain):
+    percent = round(keysize_per_chain[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(keysize_per_chain[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nSignature algorithm (ex. root) Count")
+print("------------------------------+---------")
+for stat in sorted(sig_alg):
+    sys.stdout.write(stat.ljust(30) + " " + str(sig_alg[stat]).ljust(10) + "\n")
+
+print("\nEff. host cert chain LoS  Count     Percent")
+print("-------------------------+---------+-------")
+for stat in sorted(effective_security):
+    percent = round(effective_security[stat] / total * 100, 4)
+    sys.stdout.write(str(stat).ljust(25) + " " + str(effective_security[stat]).ljust(10) + str(percent).ljust(4) + "\n")
 
 print("\nRoot CAs                                      Count     Percent")
 print("---------------------------------------------+---------+-------")
@@ -333,12 +442,6 @@ for stat in sorted(root_CA):
     percent = round(root_CA[stat] / total * 100, 4)
     sys.stdout.write(stat.ljust(45)[0:45] + " " + str(root_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n")
 
-print("\nSignature algorithm       Count     Percent")
-print("-------------------------+---------+-------")
-for stat in sorted(sig_alg):
-    percent = round(sig_alg[stat] / total * 100, 4)
-    sys.stdout.write(stat.ljust(25) + " " + str(sig_alg[stat]).ljust(10) + str(percent).ljust(4) + "\n")
-
 print("\nIntermediate CA                               Count     Percent")
 print("---------------------------------------------+---------+-------")
 for stat in sorted(intermediate_CA):