mirror of
https://github.com/mozilla/cipherscan.git
synced 2025-06-07 11:43:39 +02:00
Merge 77f326522e into ded65c40df
This commit is contained in:
Hubert Kario
commit
ed3b15fb99
263
cipherscan
263
cipherscan
@ -22,6 +22,8 @@ fi
|
||||
if [ ! -e "$CACERTS" ]; then
|
||||
echo "Warning: CA Certificates not found at $CACERTS, export CACERTS variable with location of your trust anchors" 1>&2
|
||||
fi
|
||||
# RSA ciphers are put at the end to force Google servers to accept ECDSA ciphers
|
||||
# (probably a result of a workaround for the bug in Apple implementation of ECDSA)
|
||||
CIPHERSUITE="ALL:COMPLEMENTOFALL:+aRSA"
|
||||
DEBUG=0
|
||||
VERBOSE=0
|
||||
@ -29,10 +31,19 @@ DELAY=0
|
||||
ALLCIPHERS=0
|
||||
OUTPUTFORMAT="terminal"
|
||||
TIMEOUT=10
|
||||
|
||||
# place where to put the found intermediate CA certificates and where
|
||||
# trust anchors are stored
|
||||
CAPATH=""
|
||||
SAVECRT=""
|
||||
unset ok_protocols
|
||||
declare -A ok_protocols
|
||||
unset known_certs
|
||||
declare -A known_certs
|
||||
unset cert_checksums
|
||||
declare -A cert_checksums
|
||||
|
||||
usage() {
|
||||
echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
|
||||
echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [--savecrt directory] [-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
|
||||
usage: $0 -h|--help
|
||||
|
||||
$0 attempts to connect to a target site using all the ciphersuites it knows.
|
||||
@ -46,11 +57,13 @@ Use one of the options below:
|
||||
|
||||
-a | --allciphers Test all known ciphers individually at the end.
|
||||
-b | --benchmark Activate benchmark mode.
|
||||
--capath use CAs from directory, save intermediate certificates there
|
||||
-d | --delay Pause for n seconds between connections
|
||||
-D | --debug Output ALL the information.
|
||||
-h | --help Shows this help text.
|
||||
-j | --json Output results in JSON format.
|
||||
-o | --openssl path/to/your/openssl binary you want to use.
|
||||
--savecrt path where to save untrusted and leaf certificates
|
||||
-v | --verbose Increase verbosity.
|
||||
|
||||
The rest of the arguments will be interpreted as openssl s_client argument.
|
||||
@ -74,17 +87,47 @@ debug(){
|
||||
fi
|
||||
}
|
||||
|
||||
c_hash() {
|
||||
local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null)
|
||||
for num in $(seq 0 100); do
|
||||
if [[ $1/${h}.${num} -ef $2 ]]; then
|
||||
# file already linked, ignore
|
||||
break
|
||||
fi
|
||||
if [[ ! -e $1/${h}.${num} ]]; then
|
||||
# file doesn't exist, create a link
|
||||
pushd "$1" > /dev/null
|
||||
ln -s "$2" "${h}.${num}"
|
||||
popd > /dev/null
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Connect to a target host with the selected ciphersuite
|
||||
test_cipher_on_target() {
|
||||
local sslcommand=$@
|
||||
cipher=""
|
||||
local cmnd=""
|
||||
protocols=""
|
||||
pfs=""
|
||||
previous_cipher=""
|
||||
certificates=""
|
||||
for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2"
|
||||
do
|
||||
debug echo \"Q\" \| $sslcommand $tls_version
|
||||
local tmp=$(echo "Q" | $sslcommand $tls_version 1>/dev/stdout 2>/dev/null)
|
||||
if [[ ${ok_protocols[$tls_version]} -eq 1 ]]; then
|
||||
continue
|
||||
fi
|
||||
# sslv2 client hello doesn't support SNI extension
|
||||
# in SSLv3 mode OpenSSL just ignores the setting so it's ok
|
||||
# -status exception is ignored in SSLv2, go figure
|
||||
if [ "$tls_version" == "-ssl2" ]; then
|
||||
cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand)
|
||||
else
|
||||
cmnd=$sslcommand
|
||||
fi
|
||||
debug echo \"Q\" \| $cmnd $tls_version
|
||||
local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null)
|
||||
if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then
|
||||
current_ocspstaple="True"
|
||||
else
|
||||
@ -92,18 +135,23 @@ test_cipher_on_target() {
|
||||
fi
|
||||
# filter out the OCSP server certificate
|
||||
tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp")
|
||||
current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}')
|
||||
current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}')
|
||||
current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}')
|
||||
current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}')
|
||||
if [ -z $current_pubkey ]; then
|
||||
current_pubkey=0
|
||||
fi
|
||||
current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }')
|
||||
|
||||
# session metadata
|
||||
current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp")
|
||||
current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp")
|
||||
current_protocol=$(awk '/^\s+Protocol\s+:/ {print $3; exit}' <<<"$tmp")
|
||||
current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp")
|
||||
if [ -z $current_tickethint ]; then
|
||||
current_tickethint=None
|
||||
fi
|
||||
current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
|
||||
|
||||
# certificate metadata
|
||||
current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp")
|
||||
if [ -z $current_pubkey ]; then
|
||||
current_pubkey=0
|
||||
fi
|
||||
current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\
|
||||
awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None"
|
||||
grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
current_trusted="True"
|
||||
@ -113,7 +161,88 @@ test_cipher_on_target() {
|
||||
if [ -z $current_sigalg ]; then
|
||||
current_sigalg=None
|
||||
fi
|
||||
|
||||
# collect certificate data
|
||||
current_certificates=""
|
||||
local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\
|
||||
<<<"$tmp")
|
||||
debug "server presented $certificate_count certificates"
|
||||
local i
|
||||
for ((i=0; i<$certificate_count; i=i+1 )); do
|
||||
|
||||
# extract i'th certificate
|
||||
local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 }
|
||||
/-----BEGIN CERTIFICATE-----/ { output=1 }
|
||||
output==1 { if (n==i) print }
|
||||
/-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp")
|
||||
# put the output to an array instead running awk '{print $1}'
|
||||
local cksum=($(cksum <<<"$cert"))
|
||||
# compare the values not just checksums so that eventual collision
|
||||
# doesn't mess up results
|
||||
if [[ ${known_certs[$cksum]} == $cert ]]; then
|
||||
if [ -n "${current_certificates}" ]; then
|
||||
current_certificates+=","
|
||||
fi
|
||||
current_certificates+="\"${cert_checksums[$cksum]}\""
|
||||
continue
|
||||
fi
|
||||
|
||||
# compute sha256 fingerprint of the certificate
|
||||
local sha256sum=$(${OPENSSLBIN} x509 -outform DER\
|
||||
<<<"$cert" 2>/dev/null |\
|
||||
${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}')
|
||||
|
||||
# check if it is a CA certificate
|
||||
local isCA="False"
|
||||
if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\
|
||||
grep 'CA:TRUE' >/dev/null; then
|
||||
isCA="True"
|
||||
fi
|
||||
|
||||
# build trust source for certificate verification
|
||||
local trust_source=()
|
||||
if [[ -n $CAPATH ]]; then
|
||||
trust_source=("-CApath" "$CAPATH")
|
||||
elif [[ -e $CACERTS ]]; then
|
||||
trust_source=("-CAfile" "$CACERTS")
|
||||
fi
|
||||
|
||||
# check if the certificate is actually trusted (server may present
|
||||
# unrelated certificates that are not trusted (including self
|
||||
# signed ones)
|
||||
local saved="False"
|
||||
if ${OPENSSLBIN} verify "${trust_source[@]}" \
|
||||
-untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \
|
||||
grep ': OK$' >/dev/null; then
|
||||
|
||||
# if the certificate is an intermediate CA it may be useful
|
||||
# for connecting to servers that are misconfigured so save it
|
||||
if [[ -n $CAPATH ]] && [[ $isCA == "True" ]]; then
|
||||
if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then
|
||||
echo "$cert" > "$CAPATH/${sha256sum}.pem"
|
||||
c_hash "$CAPATH" "${sha256sum}.pem"
|
||||
fi
|
||||
saved="True"
|
||||
fi
|
||||
fi
|
||||
if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then
|
||||
if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then
|
||||
echo "$cert" > "$SAVECRT/${sha256sum}.pem"
|
||||
fi
|
||||
fi
|
||||
# save the sha sum for reporting
|
||||
if [ -n "${current_certificates}" ]; then
|
||||
current_certificates+=","
|
||||
fi
|
||||
current_certificates+="\"${sha256sum}\""
|
||||
known_certs[$cksum]="$cert"
|
||||
cert_checksums[$cksum]="$sha256sum"
|
||||
done
|
||||
debug "current_certificates: $current_certificates"
|
||||
|
||||
# parsing finished, report result
|
||||
if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then
|
||||
ok_protocols["$tls_version"]=1
|
||||
# connection failed, try again with next TLS version
|
||||
continue
|
||||
else
|
||||
@ -138,6 +267,7 @@ test_cipher_on_target() {
|
||||
trusted=$current_trusted
|
||||
tickethint=$current_tickethint
|
||||
ocspstaple=$current_ocspstaple
|
||||
certificates="$current_certificates"
|
||||
# grab the cipher and PFS key size
|
||||
done
|
||||
# if cipher is empty, that means none of the TLS version worked with
|
||||
@ -182,24 +312,28 @@ bench_cipher() {
|
||||
cipherbenchms="$((t/1000/$BENCHMARKITER))"
|
||||
}
|
||||
|
||||
|
||||
# Connect to the target and retrieve the chosen cipher
|
||||
# recursively until the connection fails
|
||||
get_cipher_pref() {
|
||||
[ "$OUTPUTFORMAT" == "terminal" ] && [ $DEBUG -lt 1 ] && echo -n '.'
|
||||
local ciphersuite="$1"
|
||||
if [ -e $CACERTS ]; then
|
||||
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
|
||||
else
|
||||
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
|
||||
|
||||
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client"
|
||||
if [ -n "$CAPATH" ]; then
|
||||
sslcommand+=" -CApath $CAPATH -showcerts"
|
||||
elif [ -e $CACERTS ]; then
|
||||
sslcommand+=" -CAfile $CACERTS"
|
||||
fi
|
||||
sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
|
||||
|
||||
verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'"
|
||||
test_cipher_on_target "$sslcommand"
|
||||
local success=$?
|
||||
# If the connection succeeded with the current cipher, benchmark and store
|
||||
if [ $success -eq 0 ]; then
|
||||
cipherspref=("${cipherspref[@]}" "$result")
|
||||
pciph=$(echo $result|awk '{print $1}')
|
||||
ciphercertificates=("${ciphercertificates[@]}" "$certificates")
|
||||
pciph=($(echo $result))
|
||||
get_cipher_pref "!$pciph:$ciphersuite"
|
||||
return 0
|
||||
fi
|
||||
@ -217,7 +351,7 @@ display_results_in_terminal() {
|
||||
local ocspstaple
|
||||
local different=False
|
||||
for cipher in "${cipherspref[@]}"; do
|
||||
pciph=$(echo $cipher|awk '{print $1}')
|
||||
pciph=($(echo $cipher))
|
||||
if [ $DOBENCHMARK -eq 1 ]; then
|
||||
bench_cipher "$pciph"
|
||||
r="$ctr $cipher $cipherbenchms"
|
||||
@ -250,13 +384,13 @@ display_results_in_terminal() {
|
||||
|
||||
if [ $DOBENCHMARK -eq 1 ]; then
|
||||
if [ $different == "True" ]; then
|
||||
header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint pfs_keysize avg_handshake_microsec"
|
||||
header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint ocsp_staple pfs_keysize avg_handshake_microsec"
|
||||
else
|
||||
header="prio ciphersuite protocols pfs_keysize avg_handshake_microsec"
|
||||
fi
|
||||
else
|
||||
if [ $different == "True" ]; then
|
||||
header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint pfs_keysize"
|
||||
header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint ocsp_staple pfs_keysize"
|
||||
else
|
||||
header="prio ciphersuite protocols pfs_keysize"
|
||||
fi
|
||||
@ -268,10 +402,14 @@ display_results_in_terminal() {
|
||||
ctr=$((ctr+1))
|
||||
fi
|
||||
if [ $different == "True" ]; then
|
||||
echo $result|grep -v '(NONE)'
|
||||
if [[ $(awk '{print $3}' <<< $result) == "SSLv3,TLSv1,TLSv1.1,TLSv1.2" ]]; then
|
||||
awk '!/(NONE)/{print $1 " " $2 " " "SSLv3-TLSv1.2" " " $4 " " $5 " " $6 " " $7 " " $8 " " $9}' <<<"$result"
|
||||
else
|
||||
echo $result|grep -v '(NONE)'
|
||||
fi
|
||||
else
|
||||
# prints priority, ciphersuite, protocols and pfs_keysize
|
||||
echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}'
|
||||
awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result"
|
||||
fi
|
||||
done|column -t
|
||||
echo
|
||||
@ -288,13 +426,18 @@ display_results_in_terminal() {
|
||||
else
|
||||
echo "OCSP stapling: not supported"
|
||||
fi
|
||||
if [[ $serverside == "True" ]]; then
|
||||
echo "Server side cipher ordering"
|
||||
else
|
||||
echo "Client side cipher ordering"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
display_results_in_json() {
|
||||
# Display the results in json
|
||||
ctr=0
|
||||
echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"ciphersuite\": ["
|
||||
echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"serverside\":\"${serverside}\",\"ciphersuite\": ["
|
||||
for cipher in "${cipherspref[@]}"; do
|
||||
[ $ctr -gt 0 ] && echo -n ','
|
||||
echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\","
|
||||
@ -302,6 +445,9 @@ display_results_in_json() {
|
||||
echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"],"
|
||||
echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"],"
|
||||
echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\","
|
||||
if [[ -n $CAPATH ]]; then
|
||||
echo -n "\"certificates\":[${ciphercertificates[$ctr]}],"
|
||||
fi
|
||||
echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\","
|
||||
echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\","
|
||||
pfs=$(echo $cipher|awk '{print $8}')
|
||||
@ -312,6 +458,60 @@ display_results_in_json() {
|
||||
echo ']}'
|
||||
}
|
||||
|
||||
test_serverside_ordering() {
|
||||
|
||||
local ciphersuite=""
|
||||
local prefered=""
|
||||
# server supports only one cipher or no ciphers, so it effectively uses server side ordering...
|
||||
if [[ ${#cipherspref[@]} -lt 2 ]]; then
|
||||
serverside="True"
|
||||
return 0
|
||||
# server supports just two ciphers, so rotate them, that should be enough
|
||||
elif [[ ${#cipherspref[@]} -eq 2 ]]; then
|
||||
|
||||
local cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
|
||||
prefered="$cipher"
|
||||
ciphersuite=$cipher
|
||||
|
||||
cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
|
||||
ciphersuite+=":$cipher"
|
||||
|
||||
# server supports 3 or more ciphers, rotate all three. This is necessary because google does
|
||||
# select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or
|
||||
# ECDHE-RSA-CHACHA20-POLY1305
|
||||
else
|
||||
local cipher=$(awk '{print $1}' <<< ${cipherspref[2]})
|
||||
prefered="$cipher"
|
||||
ciphersuite="$cipher"
|
||||
|
||||
cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
|
||||
ciphersuite+=":$cipher"
|
||||
|
||||
cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
|
||||
ciphersuite+=":$cipher"
|
||||
fi
|
||||
|
||||
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client"
|
||||
if [ -n "$CAPATH" ]; then
|
||||
sslcommand+=" -CApath $CAPATH -showcerts"
|
||||
elif [ -e "$CACERTS" ]; then
|
||||
sslcommand+=" -CAfile $CACERTS"
|
||||
fi
|
||||
sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
|
||||
|
||||
test_cipher_on_target "$sslcommand"
|
||||
if [ $? -ne 0 ]; then
|
||||
serverside="True"
|
||||
else
|
||||
local selected=$(awk '{print $1}' <<< $result)
|
||||
if [[ $selected == $prefered ]]; then
|
||||
serverside="False"
|
||||
else
|
||||
serverside="True"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# UNKNOWNOPTIONS=""
|
||||
while :
|
||||
do
|
||||
@ -349,6 +549,14 @@ do
|
||||
DELAY=$2
|
||||
shift 2
|
||||
;;
|
||||
--capath)
|
||||
CAPATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--savecrt)
|
||||
SAVECRT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--) # End of all options
|
||||
shift
|
||||
break
|
||||
@ -393,10 +601,15 @@ debug "sclientargs: $SCLIENTARGS"
|
||||
|
||||
|
||||
cipherspref=();
|
||||
ciphercertificates=()
|
||||
results=()
|
||||
|
||||
# Call to the recursive loop that retrieves the cipher preferences
|
||||
get_cipher_pref $CIPHERSUITE
|
||||
unset ok_protocols
|
||||
declare -A ok_protocols
|
||||
|
||||
test_serverside_ordering
|
||||
|
||||
if [ "$OUTPUTFORMAT" == "json" ]; then
|
||||
display_results_in_json
|
||||
|
||||
509
top1m/parse_CAs.c
Normal file
509
top1m/parse_CAs.c
Normal file
@ -0,0 +1,509 @@
|
||||
/*
|
||||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
* Author: Hubert Kario - 2014
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <openssl/x509.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <openssl/x509v3.h>
|
||||
#include <openssl/stack.h>
|
||||
#include <openssl/err.h>
|
||||
#include <json-c/json.h>
|
||||
|
||||
static char* CA_TRUSTED = "./ca_trusted";
|
||||
static char* CA_ALL = "./ca_files";
|
||||
static char* CERTS_DIR = "./certs";
|
||||
|
||||
/* SSL context that knows only about trust anchors */
|
||||
SSL_CTX *trusted_only;
|
||||
/* SSL context that also has access to other CA certs */
|
||||
SSL_CTX *all_CAs;
|
||||
|
||||
// load certificate from file to a OpenSSL object
|
||||
X509 *load_cert(char *filename)
|
||||
{
|
||||
BIO* f;
|
||||
X509 *ret;
|
||||
|
||||
f = BIO_new(BIO_s_file());
|
||||
BIO_read_filename(f, filename);
|
||||
|
||||
ret = PEM_read_bio_X509_AUX(f, NULL, 0, NULL);
|
||||
if (ret == NULL)
|
||||
fprintf(stderr, "Unable to load file %s as X509 certificate\n", filename);
|
||||
|
||||
BIO_free_all(f);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
// convert sha256 to a file name, if the file exists
|
||||
// search in "all CAs" dir and "leaf certs" directories
|
||||
char *hash_to_filename(const char *hash)
|
||||
{
|
||||
char *tmp_f_name;
|
||||
size_t n;
|
||||
|
||||
n = strlen(hash) + 30;
|
||||
|
||||
// TODO error checking
|
||||
tmp_f_name = malloc(n);
|
||||
|
||||
/* first check if the file is in directory with regular certs */
|
||||
// TODO error checking
|
||||
snprintf(tmp_f_name, n, "%s/%s.pem", CERTS_DIR, hash);
|
||||
if (access(tmp_f_name, F_OK) != -1) {
|
||||
return tmp_f_name;
|
||||
}
|
||||
|
||||
snprintf(tmp_f_name, n, "%s/%s.pem", CA_ALL, hash);
|
||||
if (access(tmp_f_name, F_OK) != -1) {
|
||||
return tmp_f_name;
|
||||
}
|
||||
|
||||
// file not found
|
||||
free(tmp_f_name);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// take certificate hashes, check their validity and output json that
|
||||
// will indicate which certificate were used for verification, whatever
|
||||
// the chain was trusted and if all certificates needed for verification
|
||||
// (with the exception of root CA) were present in hashes
|
||||
int process_chain(const char **cert_hashes)
|
||||
{
|
||||
int ret;
|
||||
int rc; // return code from function
|
||||
char *f_name;
|
||||
|
||||
X509 *cert;
|
||||
X509 *x509;
|
||||
|
||||
X509_STORE *store;
|
||||
|
||||
X509_STORE_CTX *csc;
|
||||
|
||||
STACK_OF(X509) *ustack;
|
||||
STACK_OF(X509) *vstack;
|
||||
|
||||
// load certificates to temp structures
|
||||
|
||||
// first the end entity cert
|
||||
// (EE cert needs to be passed separately to OpenSSL verification context)
|
||||
f_name = hash_to_filename(cert_hashes[0]);
|
||||
if (f_name == NULL)
|
||||
return 1;
|
||||
|
||||
cert = load_cert(f_name);
|
||||
free(f_name);
|
||||
if (cert == NULL) {
|
||||
printf("can't load certificate!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
// then the intermediate certificates
|
||||
ustack = sk_X509_new_null();
|
||||
|
||||
for (int i=1; cert_hashes[i]!=NULL; i++) {
|
||||
//printf(".\n");
|
||||
f_name = hash_to_filename(cert_hashes[i]);
|
||||
if (f_name == NULL) {
|
||||
// file not found
|
||||
continue;
|
||||
}
|
||||
x509 = load_cert(f_name);
|
||||
if (x509 == NULL) {
|
||||
// loading cert failed
|
||||
continue;
|
||||
}
|
||||
sk_X509_push(ustack, x509);
|
||||
free(f_name);
|
||||
}
|
||||
|
||||
// first try with just trusted certificates
|
||||
|
||||
store = SSL_CTX_get_cert_store(trusted_only);
|
||||
if (store == NULL) {
|
||||
fprintf(stderr, "store init failed\n");
|
||||
return 1;
|
||||
}
|
||||
X509_STORE_set_flags(store, X509_V_FLAG_TRUSTED_FIRST);
|
||||
|
||||
csc = X509_STORE_CTX_new();
|
||||
|
||||
ret = X509_STORE_CTX_init(csc, store, cert, ustack);
|
||||
if (ret != 1) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
ret = X509_verify_cert(csc);
|
||||
|
||||
if (ret != 1) {
|
||||
// printf("%s\n", X509_verify_cert_error_string(csc->error));
|
||||
} else {
|
||||
// chain is complete, output certificate hashes
|
||||
printf("{\"chain\":\"complete\",\"certificates\":[");
|
||||
vstack = X509_STORE_CTX_get_chain(csc);
|
||||
for(int i=0; i<sk_X509_num(vstack); i++) {
|
||||
X509 *c = sk_X509_value(vstack, i);
|
||||
|
||||
const EVP_MD *digest;
|
||||
unsigned char md[EVP_MAX_MD_SIZE];
|
||||
int n;
|
||||
digest = EVP_get_digestbyname("sha256");
|
||||
X509_digest(c, digest, md, &n);
|
||||
printf("\"");
|
||||
for(int i=0; i<n; i++) {
|
||||
printf("%02x", md[i]);
|
||||
}
|
||||
printf("\"");
|
||||
if (i+1 < sk_X509_num(vstack)) {
|
||||
printf(",");
|
||||
}
|
||||
}
|
||||
printf("]}");
|
||||
X509_STORE_CTX_free(csc);
|
||||
sk_X509_pop_free(ustack, X509_free);
|
||||
X509_free(cert);
|
||||
return 0;
|
||||
}
|
||||
X509_STORE_CTX_free(csc);
|
||||
|
||||
// validation failed with just the trust anchors, retry with all
|
||||
// known intermediate certificates
|
||||
|
||||
store = SSL_CTX_get_cert_store(all_CAs);
|
||||
if (store == NULL) {
|
||||
fprintf(stderr, "store init failed\n");
|
||||
return 1;
|
||||
}
|
||||
X509_STORE_set_flags(store, X509_V_FLAG_TRUSTED_FIRST);
|
||||
|
||||
csc = X509_STORE_CTX_new();
|
||||
|
||||
ret = X509_STORE_CTX_init(csc, store, cert, ustack);
|
||||
if (ret != 1) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
ret = X509_verify_cert(csc);
|
||||
if (ret != 1) {
|
||||
// certificate untrusted
|
||||
printf("{\"chain\":\"untrusted\"}");
|
||||
} else {
|
||||
// chain successfully verified using all certificates,
|
||||
// print all the certs used to verify it
|
||||
printf("{\"chain\":\"incomplete\",\"certificates\":[");
|
||||
vstack = X509_STORE_CTX_get_chain(csc);
|
||||
for(int i=0; i<sk_X509_num(vstack); i++) {
|
||||
X509 *c = sk_X509_value(vstack, i);
|
||||
|
||||
const EVP_MD *digest;
|
||||
unsigned char md[EVP_MAX_MD_SIZE];
|
||||
int n;
|
||||
digest = EVP_get_digestbyname("sha256");
|
||||
X509_digest(c, digest, md, &n);
|
||||
printf("\"");
|
||||
for(int i=0; i<n; i++) {
|
||||
printf("%02x", md[i]);
|
||||
}
|
||||
printf("\"");
|
||||
if (i+1 < sk_X509_num(vstack)) {
|
||||
printf(",");
|
||||
}
|
||||
}
|
||||
printf("]}");
|
||||
}
|
||||
|
||||
X509_STORE_CTX_free(csc);
|
||||
sk_X509_pop_free(ustack, X509_free);
|
||||
X509_free(cert);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// check if array of strings in json object is the same or not
|
||||
int string_array_cmp(struct json_object *a, struct json_object *b)
|
||||
{
|
||||
if (json_object_get_type(a) != json_type_array)
|
||||
return -1; // wrong type
|
||||
|
||||
if (json_object_get_type(b) != json_type_array)
|
||||
return -1; // wrong type
|
||||
|
||||
if (json_object_array_length(a) != json_object_array_length(b))
|
||||
return 1;
|
||||
|
||||
for (int i=0; i<json_object_array_length(a); i++) {
|
||||
struct json_object *s_a, *s_b;
|
||||
const char *str_a, *str_b;
|
||||
|
||||
s_a = json_object_array_get_idx(a, i);
|
||||
if (json_object_get_type(s_a) != json_type_string)
|
||||
return -1; // wrong type
|
||||
|
||||
s_b = json_object_array_get_idx(b, i);
|
||||
if (json_object_get_type(s_b) != json_type_string)
|
||||
return -1; // wrong type
|
||||
|
||||
str_a = json_object_get_string(s_a);
|
||||
str_b = json_object_get_string(s_b);
|
||||
if (str_a == NULL && str_b == NULL)
|
||||
continue;
|
||||
|
||||
if (str_a == NULL || str_b == NULL)
|
||||
return 1;
|
||||
|
||||
if (strcmp(str_a, str_b) != 0)
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// add a list of new strings (hashes) to a list of known strings, if they are
|
||||
// indeed new, don't do anything if they are already in the known set
|
||||
int register_known_chains(struct json_object ***known, struct json_object *new)
|
||||
{
|
||||
int rc;
|
||||
|
||||
if (*known == NULL) {
|
||||
*known = calloc(sizeof(struct json_object**), 2);
|
||||
(*known)[0] = new;
|
||||
return 0; // it's a new one
|
||||
}
|
||||
|
||||
int i;
|
||||
for (i=0; (*known)[i] != NULL; i++) {
|
||||
rc = string_array_cmp((*known)[i], new);
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "error in string_array_cmp\n");
|
||||
}
|
||||
if (string_array_cmp((*known)[i], new) == 0) {
|
||||
return 1; // we've seen it before
|
||||
}
|
||||
}
|
||||
|
||||
// add it to known objects
|
||||
*known = realloc(*known, sizeof(struct json_object **)*(i+2));
|
||||
// TODO handle errors
|
||||
(*known)[i] = new;
|
||||
(*known)[i+1] = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct json_object *read_json_from_file(char *filename)
|
||||
{
|
||||
json_tokener *tok;
|
||||
|
||||
struct json_object *obj = NULL;
|
||||
int ret = 0;
|
||||
int rc;
|
||||
size_t len = 8192;
|
||||
char buffer[len];
|
||||
char *start;
|
||||
int i;
|
||||
enum json_tokener_error jerr;
|
||||
|
||||
int fd;
|
||||
|
||||
fd = open(filename, 0);
|
||||
if (fd < 0) {
|
||||
ret = 1;
|
||||
goto err;
|
||||
}
|
||||
// skip garbage at the beginning of file (old `cipherscan` versions
|
||||
// sometimes did put `popd` and pushd` output in the json file)
|
||||
do {
|
||||
rc = read(fd, buffer, 1);
|
||||
} while (buffer[0] != '{' || rc < 0);
|
||||
if (rc >= 0) {
|
||||
lseek(fd, -1, SEEK_CUR);
|
||||
}
|
||||
|
||||
// parse the json object from the file
|
||||
tok = json_tokener_new();
|
||||
do {
|
||||
rc = read(fd, buffer, len);
|
||||
if (rc < 0)
|
||||
break;
|
||||
obj = json_tokener_parse_ex(tok, buffer, rc);
|
||||
} while ((jerr = json_tokener_get_error(tok)) == json_tokener_continue);
|
||||
|
||||
if (jerr != json_tokener_success){
|
||||
fprintf(stderr, "error in file %s, line: %s\n", filename, buffer);
|
||||
}
|
||||
|
||||
tok_free:
|
||||
|
||||
json_tokener_free(tok);
|
||||
|
||||
close_fd:
|
||||
close(fd);
|
||||
|
||||
err:
|
||||
if (ret) {
|
||||
fprintf(stderr, "error while reading file: %i", ret);
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
// process all ciphersuites one by one from a given host results file
|
||||
int process_host_results(char *filename)
|
||||
{
|
||||
int fd;
|
||||
int ret = 0;
|
||||
int rc;
|
||||
size_t sz;
|
||||
size_t alloc_size = 64 * 1024;
|
||||
const char *str;
|
||||
struct json_object *root;
|
||||
struct json_object *ciphers;
|
||||
struct json_object *current;
|
||||
struct json_object *certificates;
|
||||
|
||||
struct json_object **known_chains;
|
||||
known_chains = malloc(sizeof(struct json_object*) * 1);
|
||||
known_chains[0] = NULL;
|
||||
|
||||
struct lh_table *table;
|
||||
enum json_type obj_t;
|
||||
json_bool j_rc;
|
||||
|
||||
root = read_json_from_file(filename);
|
||||
if (root == NULL) {
|
||||
ret = 1;
|
||||
goto err;
|
||||
}
|
||||
|
||||
obj_t = json_object_get_type(root);
|
||||
str = json_type_to_name(obj_t);
|
||||
|
||||
j_rc = json_object_object_get_ex(root, "ciphersuite", &ciphers);
|
||||
if (j_rc == FALSE) {
|
||||
ret = 1;
|
||||
goto json_free;
|
||||
}
|
||||
|
||||
// ok, we've got the ciphersuite part, we can print the json header for
|
||||
// the host file
|
||||
printf("{\"host\":\"%s\",\"chains\":[", filename);
|
||||
|
||||
int first_printed=0;
|
||||
for(int i=0; i < json_object_array_length(ciphers); i++) {
|
||||
current = json_object_array_get_idx(ciphers, i);
|
||||
//printf("\t[%i]:\n", i);
|
||||
j_rc = json_object_object_get_ex(current, "certificates", &certificates);
|
||||
if (j_rc == FALSE)
|
||||
continue;
|
||||
|
||||
const char** certs;
|
||||
certs = calloc(sizeof(const char*), json_object_array_length(certificates) + 1);
|
||||
int j;
|
||||
for (j=0; j < json_object_array_length(certificates); j++) {
|
||||
certs[j] = json_object_get_string(json_object_array_get_idx(certificates, j));
|
||||
//printf("\t\t\t%s\n", certs[j]);
|
||||
}
|
||||
rc = register_known_chains(&known_chains, certificates);
|
||||
//printf("\t\t%i\n", rc);
|
||||
|
||||
if (rc == 0 && j > 0) {
|
||||
if (first_printed != 0)
|
||||
printf(",");
|
||||
if (process_chain(certs) != 0) {
|
||||
fprintf(stderr, "error while processing chains!\n");
|
||||
} else {
|
||||
first_printed = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// DEBUG, print whole json "object" object
|
||||
//json_object_object_foreach(current, key, val) {
|
||||
// str = json_object_to_json_string(val);
|
||||
// printf("\t\t%s: %s\n", key, str);
|
||||
//}
|
||||
|
||||
free(certs);
|
||||
}
|
||||
printf("]}");
|
||||
|
||||
json_free:
|
||||
json_object_put(root);
|
||||
|
||||
err:
|
||||
free(known_chains);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int ret;
|
||||
|
||||
DIR *dirp;
|
||||
struct dirent *direntp;
|
||||
|
||||
char buffer[8192] = {};
|
||||
|
||||
SSL_load_error_strings();
|
||||
SSL_library_init();
|
||||
|
||||
/* init trust stores with certificate locations */
|
||||
trusted_only = SSL_CTX_new(SSLv23_method());
|
||||
if (trusted_only == NULL) {
|
||||
ERR_print_errors_fp(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
ret = SSL_CTX_load_verify_locations(trusted_only, NULL, CA_TRUSTED);
|
||||
if (ret != 1) {
|
||||
ERR_print_errors_fp(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
all_CAs = SSL_CTX_new(SSLv23_method());
|
||||
if (all_CAs == NULL) {
|
||||
ERR_print_errors_fp(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
ret = SSL_CTX_load_verify_locations(all_CAs, NULL, CA_ALL);
|
||||
if (ret != 1) {
|
||||
ERR_print_errors_fp(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* traverse the result directory, check all files in turn */
|
||||
dirp=opendir("results");
|
||||
while((direntp=readdir(dirp)) != NULL) {
|
||||
if (strcmp(direntp->d_name, ".") == 0)
|
||||
continue;
|
||||
if (strcmp(direntp->d_name, "..") == 0)
|
||||
continue;
|
||||
|
||||
snprintf(buffer, 8191, "results/%s", direntp->d_name);
|
||||
|
||||
ret = process_host_results(buffer);
|
||||
if (ret == 1) {
|
||||
fprintf(stderr, "error while processing %s\n", buffer);
|
||||
}
|
||||
if (ret == 0)
|
||||
printf("\n");
|
||||
}
|
||||
closedir(dirp);
|
||||
|
||||
/* clean up */
|
||||
SSL_CTX_free(trusted_only);
|
||||
SSL_CTX_free(all_CAs);
|
||||
all_CAs = NULL;
|
||||
trusted_only = NULL;
|
||||
|
||||
return ret;
|
||||
}
|
||||
328
top1m/parse_CAs.py
Normal file
328
top1m/parse_CAs.py
Normal file
@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# Author: Hubert Kario - 2014
|
||||
|
||||
from __future__ import division
|
||||
|
||||
path = "./results/"
|
||||
ca_certs_path = "./ca_files"
|
||||
certs_path = "./certs"
|
||||
|
||||
""" only root CAs, no cached intermediate certs """
|
||||
trust_path = "./ca_trusted"
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
import os
|
||||
from OpenSSL import crypto
|
||||
|
||||
invocations = defaultdict(int)
|
||||
|
||||
total = 0
|
||||
hosts = 0
|
||||
chains = defaultdict(int)
|
||||
chain_len = defaultdict(int)
|
||||
keysize = defaultdict(int)
|
||||
keysize_per_chain = defaultdict(int)
|
||||
root_CA = defaultdict(int)
|
||||
sig_alg = defaultdict(int)
|
||||
intermediate_CA = defaultdict(int)
|
||||
effective_security = defaultdict(int)
|
||||
|
||||
subject_hashes = {}
|
||||
issuer_hashes = {}
|
||||
|
||||
def get_cert_subject_name(cert):
|
||||
subject = cert.get_subject()
|
||||
commonName = None
|
||||
organization = None
|
||||
|
||||
for elem,val in subject.get_components():
|
||||
if elem == "CN" and commonName is None:
|
||||
commonName = val
|
||||
if elem == "O" and organization is None:
|
||||
organization = val
|
||||
|
||||
s_hash = "(" + ("%0.8X" % subject.hash()).lower() + ") "
|
||||
|
||||
if commonName is not None:
|
||||
return s_hash + commonName
|
||||
elif organization is not None:
|
||||
return s_hash + organization
|
||||
else:
|
||||
return s_hash
|
||||
|
||||
def get_path_for_hash(cert_hash):
|
||||
f_name = certs_path + '/' + cert_hash + '.pem'
|
||||
if not os.path.exists(f_name):
|
||||
f_name = ca_certs_path + '/' + cert_hash + '.pem'
|
||||
if not os.path.exists(f_name):
|
||||
#print "File with hash " + c_hash + " is missing!"
|
||||
return None
|
||||
return f_name
|
||||
|
||||
""" convert RSA and DSA key sizes to estimated Level of security """
|
||||
def rsa_key_size_to_los(size):
|
||||
if size < 760:
|
||||
return 40
|
||||
elif size < 1020:
|
||||
return 64
|
||||
elif size < 2040:
|
||||
return 80
|
||||
elif size < 3068:
|
||||
return 112
|
||||
elif size < 4094:
|
||||
return 128
|
||||
elif size < 7660:
|
||||
return 152
|
||||
elif size < 15300:
|
||||
return 192
|
||||
else:
|
||||
return 256
|
||||
|
||||
def sig_alg_to_los(name):
|
||||
if 'MD5' in name.upper():
|
||||
return 64
|
||||
elif 'SHA1' in name.upper():
|
||||
return 80
|
||||
elif 'SHA224' in name.upper():
|
||||
return 112
|
||||
elif 'SHA256' in name.upper():
|
||||
return 128
|
||||
elif 'SHA384' in name.upper():
|
||||
return 192
|
||||
elif 'SHA512' in name.upper():
|
||||
return 256
|
||||
else:
|
||||
raise UnknownSigAlgError
|
||||
|
||||
def collect_key_sizes(file_names):
|
||||
|
||||
tmp_keysize = {}
|
||||
|
||||
""" don't collect signature alg for the self signed root """
|
||||
with open(file_names[-1]) as cert_file:
|
||||
cert_pem = cert_file.read()
|
||||
|
||||
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
|
||||
|
||||
pubkey = cert.get_pubkey()
|
||||
if pubkey.type() == crypto.TYPE_RSA:
|
||||
keysize['RSA ' + str(pubkey.bits())] += 1
|
||||
tmp_keysize['RSA ' + str(pubkey.bits())] = 1
|
||||
security_level = rsa_key_size_to_los(pubkey.bits())
|
||||
elif pubkey.type() == crypto.TYPE_DSA:
|
||||
keysize['DSA ' + str(pubkey.bits())] += 1
|
||||
tmp_keysize['DSA ' + str(pubkey.bits())] = 1
|
||||
security_level = rsa_key_size_to_los(pubkey.bits())
|
||||
elif pubkey.type() == 408:
|
||||
keysize['ECDSA ' + str(pubkey.bits())] += 1
|
||||
tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1
|
||||
security_level = pubkey.bits()/2
|
||||
else:
|
||||
keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1
|
||||
security_level = 0
|
||||
|
||||
root_CA[get_cert_subject_name(cert)] += 1
|
||||
|
||||
""" exclude the self signed root and server cert from stats """
|
||||
for f_name in file_names[1:-1]:
|
||||
with open(f_name) as cert_file:
|
||||
cert_pem = cert_file.read()
|
||||
|
||||
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
|
||||
|
||||
pubkey = cert.get_pubkey()
|
||||
if pubkey.type() == crypto.TYPE_RSA:
|
||||
keysize['RSA ' + str(pubkey.bits())] += 1
|
||||
tmp_keysize['RSA ' + str(pubkey.bits())] = 1
|
||||
c_key_level = rsa_key_size_to_los(pubkey.bits())
|
||||
elif pubkey.type() == crypto.TYPE_DSA:
|
||||
keysize['DSA ' + str(pubkey.bits())] += 1
|
||||
tmp_keysize['DSA ' + str(pubkey.bits())] = 1
|
||||
c_key_level = rsa_key_size_to_los(pubkey.bits())
|
||||
elif pubkey.type() == 408:
|
||||
keysize['ECDSA ' + str(pubkey.bits())] += 1
|
||||
tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1
|
||||
c_key_level = pubkey.bits() / 2
|
||||
else:
|
||||
keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1
|
||||
c_key_level = 0
|
||||
|
||||
if security_level > c_key_level:
|
||||
security_level = c_key_level
|
||||
|
||||
sig_alg[cert.get_signature_algorithm()] += 1
|
||||
c_sig_level = sig_alg_to_los(cert.get_signature_algorithm())
|
||||
if security_level > c_sig_level:
|
||||
security_level = c_sig_level
|
||||
|
||||
intermediate_CA[get_cert_subject_name(cert)] += 1
|
||||
|
||||
for key_s in tmp_keysize:
|
||||
keysize_per_chain[key_s] += 1
|
||||
|
||||
# XXX doesn't handle the situation in which the CA uses its certificate
|
||||
# for a web server properly
|
||||
with open(file_names[0]) as cert_file:
|
||||
cert_pem = cert_file.read()
|
||||
|
||||
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
|
||||
|
||||
pubkey = cert.get_pubkey()
|
||||
if pubkey.type() == crypto.TYPE_RSA:
|
||||
c_key_level = rsa_key_size_to_los(pubkey.bits())
|
||||
elif pubkey.type() == crypto.TYPE_DSA:
|
||||
c_key_level = rsa_key_size_to_los(pubkey.bits())
|
||||
elif pubkey.type() == 408:
|
||||
c_key_level = pubkey.bits() / 2
|
||||
else:
|
||||
c_key_level = 0
|
||||
|
||||
if security_level > c_key_level:
|
||||
security_level = c_key_level
|
||||
|
||||
c_sig_level = sig_alg_to_los(cert.get_signature_algorithm())
|
||||
if security_level > c_sig_level:
|
||||
security_level = c_sig_level
|
||||
|
||||
effective_security[security_level] += 1
|
||||
|
||||
|
||||
with open("parsed") as res_file:
|
||||
for line in res_file:
|
||||
try:
|
||||
res = json.loads(line)
|
||||
except ValueError as e:
|
||||
print "can't process line: " + line
|
||||
continue
|
||||
|
||||
f=res
|
||||
|
||||
try:
|
||||
server_chain_trusted = False
|
||||
server_chain_complete = False
|
||||
server_chains = []
|
||||
valid = False
|
||||
|
||||
""" Keep certificates in memory for a given file """
|
||||
known_certs = {}
|
||||
|
||||
if not "chains" in f:
|
||||
continue
|
||||
|
||||
results = f["chains"]
|
||||
|
||||
""" discard hosts with empty results """
|
||||
if len(results) < 1:
|
||||
continue
|
||||
|
||||
""" loop over list of ciphers """
|
||||
for entry in results:
|
||||
|
||||
""" skip invalid results """
|
||||
if not 'chain' in entry:
|
||||
continue
|
||||
|
||||
valid = True
|
||||
|
||||
if entry['chain'] == "untrusted":
|
||||
continue
|
||||
|
||||
if entry['chain'] == "complete":
|
||||
server_chain_complete = True
|
||||
server_chain_trusted = True
|
||||
|
||||
if entry['chain'] == "incomplete":
|
||||
server_chain_trusted = True
|
||||
|
||||
server_chains += [entry['certificates']]
|
||||
|
||||
if server_chain_trusted:
|
||||
if server_chain_complete:
|
||||
chains["complete"] += 1
|
||||
print "complete: " + f['host']
|
||||
else:
|
||||
chains["incomplete"] += 1
|
||||
print "incomplete: " + f['host']
|
||||
else:
|
||||
chains["untrusted"] += 1
|
||||
print "untrusted: " + f['host']
|
||||
|
||||
if valid:
|
||||
hosts += 1
|
||||
|
||||
for chain in server_chains:
|
||||
f_names = []
|
||||
for hash in chain:
|
||||
path = get_path_for_hash(hash)
|
||||
f_names += [path]
|
||||
|
||||
collect_key_sizes(f_names)
|
||||
chain_len[str(len(chain))] += 1
|
||||
if len(chain) == 1:
|
||||
sys.stderr.write("file with chain 1 long: " + line)
|
||||
total += 1
|
||||
except TypeError as e:
|
||||
|
||||
sys.stderr.write("can't process: " + line)
|
||||
continue
|
||||
|
||||
""" Display stats """
|
||||
#print "openssl invocations: " + str(invocations["openssl"])
|
||||
|
||||
print "Statistics from " + str(total) + " chains provided by " + str(hosts) + " hosts"
|
||||
|
||||
print("\nServer provided chains Count Percent")
|
||||
print("-------------------------+---------+-------")
|
||||
for stat in sorted(chains):
|
||||
percent = round(chains[stat] / hosts * 100, 4)
|
||||
sys.stdout.write(stat.ljust(25) + " " + str(chains[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nTrusted chain statistics")
|
||||
print("========================")
|
||||
|
||||
|
||||
print("\nChain length Count Percent")
|
||||
print("-------------------------+---------+-------")
|
||||
for stat in sorted(chain_len):
|
||||
percent = round(chain_len[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(25) + " " + str(chain_len[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nCA key size in chains Count")
|
||||
print("-------------------------+---------")
|
||||
for stat in sorted(keysize):
|
||||
sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + "\n")
|
||||
|
||||
print("\nChains with CA key Count Percent")
|
||||
print("-------------------------+---------+-------")
|
||||
for stat in sorted(keysize_per_chain):
|
||||
percent = round(keysize_per_chain[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(25) + " " + str(keysize_per_chain[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nSignature algorithm (ex. root) Count")
|
||||
print("------------------------------+---------")
|
||||
for stat in sorted(sig_alg):
|
||||
sys.stdout.write(stat.ljust(30) + " " + str(sig_alg[stat]).ljust(10) + "\n")
|
||||
|
||||
print("\nEff. host cert chain LoS Count Percent")
|
||||
print("-------------------------+---------+-------")
|
||||
for stat in sorted(effective_security):
|
||||
percent = round(effective_security[stat] / total * 100, 4)
|
||||
sys.stdout.write(str(stat).ljust(25) + " " + str(effective_security[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nRoot CAs Count Percent")
|
||||
print("---------------------------------------------+---------+-------")
|
||||
for stat in sorted(root_CA):
|
||||
percent = round(root_CA[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(45)[0:45] + " " + str(root_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nIntermediate CA Count Percent")
|
||||
print("---------------------------------------------+---------+-------")
|
||||
for stat in sorted(intermediate_CA):
|
||||
percent = round(intermediate_CA[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(45)[0:45] + " " + str(intermediate_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
@ -14,10 +14,47 @@ import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
import os
|
||||
import re
|
||||
|
||||
def natural_sort(l):
|
||||
convert = lambda text: int(text) if text.isdigit() else text.lower()
|
||||
alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
|
||||
return sorted(l, key = alphanum_key)
|
||||
|
||||
""" list of ciphers offerred by Firefox 29 by default """
|
||||
firefox_ciphers=[
|
||||
'ECDHE-ECDSA-AES128-GCM-SHA256',
|
||||
'ECDHE-RSA-AES128-GCM-SHA256',
|
||||
'ECDHE-ECDSA-AES256-SHA',
|
||||
'ECDHE-ECDSA-AES128-SHA',
|
||||
'ECDHE-RSA-AES128-SHA',
|
||||
'ECDHE-RSA-AES256-SHA',
|
||||
'ECDHE-RSA-DES-CBC3-SHA',
|
||||
'ECDHE-ECDSA-RC4-SHA',
|
||||
'ECDHE-RSA-RC4-SHA',
|
||||
'DHE-RSA-AES128-SHA',
|
||||
'DHE-DSS-AES128-SHA',
|
||||
'DHE-RSA-CAMELLIA128-SHA',
|
||||
'DHE-RSA-AES256-SHA',
|
||||
'DHE-DSS-AES256-SHA',
|
||||
'DHE-RSA-CAMELLIA256-SHA',
|
||||
'EDH-RSA-DES-CBC3-SHA',
|
||||
'AES128-SHA',
|
||||
'CAMELLIA128-SHA',
|
||||
'AES256-SHA',
|
||||
'CAMELLIA256-SHA',
|
||||
'DES-CBC3-SHA',
|
||||
'RC4-SHA',
|
||||
'RC4-MD5']
|
||||
|
||||
report_untrused=False
|
||||
|
||||
cipherstats = defaultdict(int)
|
||||
FF_RC4_Only_cipherstats = defaultdict(int)
|
||||
FF_RC4_preferred_cipherstats = defaultdict(int)
|
||||
FF_incompatible_cipherstats = defaultdict(int)
|
||||
FF_selected_cipherstats = defaultdict(int)
|
||||
cipherordering = defaultdict(int)
|
||||
pfsstats = defaultdict(int)
|
||||
protocolstats = defaultdict(int)
|
||||
handshakestats = defaultdict(int)
|
||||
@ -38,6 +75,7 @@ for r,d,flist in os.walk(path):
|
||||
tempdsakeystats = {}
|
||||
tempsigstats = {}
|
||||
tempticketstats = {}
|
||||
""" supported ciphers by the server under scan """
|
||||
tempcipherstats = {}
|
||||
ciphertypes = 0
|
||||
AESGCM = False
|
||||
@ -46,6 +84,13 @@ for r,d,flist in os.walk(path):
|
||||
DES3 = False
|
||||
CAMELLIA = False
|
||||
RC4 = False
|
||||
""" the following depends on FF_compat, so by default it can be True """
|
||||
RC4_Only_FF = True
|
||||
FF_compat = False
|
||||
temp_FF_incompat = {}
|
||||
list_of_ciphers = []
|
||||
FF_RC4_Pref = None
|
||||
FF_selected = None
|
||||
ADH = False
|
||||
DHE = False
|
||||
AECDH = False
|
||||
@ -85,11 +130,25 @@ for r,d,flist in os.walk(path):
|
||||
if 'False' in entry['trusted'] and report_untrused == False:
|
||||
continue
|
||||
|
||||
list_of_ciphers.append(entry['cipher'])
|
||||
|
||||
# check if the advertised ciphers are not effectively RC4 Only
|
||||
# for firefox or incompatible with firefox
|
||||
if entry['cipher'] in firefox_ciphers:
|
||||
# if this is first cipher and we already are getting RC4
|
||||
# then it means that RC4 is preferred
|
||||
FF_compat = True
|
||||
if not 'RC4' in entry['cipher']:
|
||||
RC4_Only_FF = False
|
||||
else:
|
||||
temp_FF_incompat[entry['cipher']] = 1
|
||||
|
||||
""" store the ciphers supported """
|
||||
if 'ADH' in entry['cipher'] or 'AECDH' in entry['cipher']:
|
||||
ciphertypes += 1
|
||||
name = "z:" + entry['cipher']
|
||||
tempcipherstats[name] = 1
|
||||
tempcipherstats['Insecure'] = 1
|
||||
elif 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']:
|
||||
if not AESGCM:
|
||||
AESGCM = True
|
||||
@ -114,10 +173,15 @@ for r,d,flist in os.walk(path):
|
||||
if not CHACHA20:
|
||||
ciphertypes += 1
|
||||
CHACHA20 = True
|
||||
elif 'IDEA' in entry['cipher'] or 'SEED' in entry['cipher']:
|
||||
ciphertypes += 1
|
||||
name = "y:" + entry['cipher']
|
||||
tempcipherstats[name] = 1
|
||||
else:
|
||||
ciphertypes += 1
|
||||
name = "z:" + entry['cipher']
|
||||
tempcipherstats[name] = 1
|
||||
tempcipherstats['Insecure'] = 1
|
||||
|
||||
""" store key handshake methods """
|
||||
if 'ECDHE' in entry['cipher']:
|
||||
@ -138,7 +202,7 @@ for r,d,flist in os.walk(path):
|
||||
RSA = True
|
||||
|
||||
""" save the key size """
|
||||
if 'ECDSA' in entry['cipher']:
|
||||
if 'ECDSA' in entry['cipher'] or 'ECDH-RSA' in entry['cipher']:
|
||||
ECDSA = True
|
||||
tempecckeystats[entry['pubkey'][0]] = 1
|
||||
elif 'DSS' in entry['cipher']:
|
||||
@ -206,6 +270,32 @@ for r,d,flist in os.walk(path):
|
||||
if dualstack:
|
||||
dsarsastack += 1
|
||||
|
||||
""" save cipher ordering """
|
||||
if 'serverside' in results:
|
||||
if results['serverside'] == "False":
|
||||
cipherordering['Client side'] += 1
|
||||
else:
|
||||
cipherordering['Server side'] += 1
|
||||
else:
|
||||
cipherordering['Unknown'] += 1
|
||||
|
||||
""" simulate handshake with Firefox """
|
||||
if FF_compat:
|
||||
if 'serverside' in results and results['serverside'] == "False":
|
||||
for cipher in firefox_ciphers:
|
||||
if cipher in list_of_ciphers:
|
||||
FF_selected = cipher
|
||||
if 'RC4' in cipher:
|
||||
FF_RC4_Pref = True
|
||||
break
|
||||
else:
|
||||
for cipher in list_of_ciphers:
|
||||
if cipher in firefox_ciphers:
|
||||
FF_selected = cipher
|
||||
if 'RC4' in cipher:
|
||||
FF_RC4_Pref = True
|
||||
break
|
||||
|
||||
for s in tempsigstats:
|
||||
sigalg[s] += 1
|
||||
|
||||
@ -256,6 +346,27 @@ for r,d,flist in os.walk(path):
|
||||
cipherstats['RC4 forced in TLS1.1+'] += 1
|
||||
cipherstats['RC4 Preferred'] += 1
|
||||
|
||||
if FF_compat:
|
||||
if 'ECDHE' in FF_selected:
|
||||
FF_selected_cipherstats['x:ECDHE'] += 1
|
||||
elif 'DHE' in FF_selected or 'EDH' in FF_selected:
|
||||
FF_selected_cipherstats['x:DHE'] += 1
|
||||
else:
|
||||
FF_selected_cipherstats['x:kRSA'] += 1
|
||||
FF_selected_cipherstats[FF_selected] += 1
|
||||
if RC4_Only_FF and ciphertypes != 1:
|
||||
cipherstats['x:FF 29 RC4 Only'] += 1
|
||||
for cipher in temp_FF_incompat:
|
||||
FF_RC4_Only_cipherstats[cipher] += 1
|
||||
if FF_RC4_Pref and not 'RC4' in results['ciphersuite'][0]['cipher']:
|
||||
cipherstats['x:FF 29 RC4 Preferred'] += 1
|
||||
for cipher in temp_FF_incompat:
|
||||
FF_RC4_preferred_cipherstats[cipher] += 1
|
||||
else:
|
||||
cipherstats['x:FF 29 incompatible'] += 1
|
||||
for cipher in temp_FF_incompat:
|
||||
FF_incompatible_cipherstats[cipher] += 1
|
||||
|
||||
for cipher in tempcipherstats:
|
||||
cipherstats[cipher] += 1
|
||||
|
||||
@ -309,6 +420,13 @@ for r,d,flist in os.walk(path):
|
||||
#if total % 1999 == 0:
|
||||
# break
|
||||
|
||||
""" The 'x:FF 29 RC4 Preferred' counts only sites that effectively prefer
|
||||
RC4 when using FF, to make reporting more readable, sum it with sites
|
||||
that do that for all ciphers"""
|
||||
|
||||
if "x:FF 29 RC4 Preferred" in cipherstats and "RC4 Preferred" in cipherstats:
|
||||
cipherstats['x:FF 29 RC4 Preferred'] += cipherstats['RC4 Preferred']
|
||||
|
||||
print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total)
|
||||
if report_untrused == False:
|
||||
print("Stats only from connections that did provide valid certificates")
|
||||
@ -321,6 +439,36 @@ for stat in sorted(cipherstats):
|
||||
percent = round(cipherstats[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nCipher ordering Count Percent")
|
||||
print("-------------------------+---------+-------")
|
||||
for stat in sorted(cipherordering):
|
||||
percent = round(cipherordering[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(25) + " " + str(cipherordering[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nFF 29 selected ciphers Count Percent")
|
||||
print("-----------------------------+---------+------")
|
||||
for stat in sorted(FF_selected_cipherstats):
|
||||
percent = round(FF_selected_cipherstats[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(30) + " " + str(FF_selected_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nFF 29 RC4 Only other ciphers Count Percent")
|
||||
print("-----------------------------+---------+------")
|
||||
for stat in sorted(FF_RC4_Only_cipherstats):
|
||||
percent = round(FF_RC4_Only_cipherstats[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_Only_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nFF 29 RC4 pref other ciphers Count Percent")
|
||||
print("-----------------------------+---------+------")
|
||||
for stat in sorted(FF_RC4_preferred_cipherstats):
|
||||
percent = round(FF_RC4_preferred_cipherstats[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_preferred_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nFF 29 incompatible ciphers Count Percent")
|
||||
print("-----------------------------+---------+------")
|
||||
for stat in sorted(FF_incompatible_cipherstats):
|
||||
percent = round(FF_incompatible_cipherstats[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(30) + " " + str(FF_incompatible_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
|
||||
|
||||
print("\nSupported Handshakes Count Percent")
|
||||
print("-------------------------+---------+-------")
|
||||
for stat in sorted(handshakestats):
|
||||
@ -340,7 +488,7 @@ for stat in sorted(pfsstats):
|
||||
|
||||
print("\nTLS session ticket hint Count Percent ")
|
||||
print("-------------------------+---------+--------")
|
||||
for stat in sorted(tickethint):
|
||||
for stat in natural_sort(tickethint):
|
||||
percent = round(tickethint[stat] / total * 100, 4)
|
||||
sys.stdout.write(stat.ljust(25) + " " + str(tickethint[stat]).ljust(10) + str(percent).ljust(9) + "\n")
|
||||
|
||||
|
||||
47
top1m/process-certificate-statistics.sh
Executable file
47
top1m/process-certificate-statistics.sh
Executable file
@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ ! -d ./ca_files ]; then
|
||||
echo "Directory with collected CA certificates missing!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d ./ca_trusted ]; then
|
||||
echo "Directory with just trust anchors missing!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d ./certs ]; then
|
||||
echo "Directory with certificates missing!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! ls -f ./ca_files/????????.? > /dev/null; then
|
||||
echo "CA certificates directory not hashed properly (use c_rehash)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! ls -f ./ca_trusted/????????.? > /dev/null; then
|
||||
echo "Directory with trust anchors not hashed properly (use c_rehash)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d ./results ]; then
|
||||
echo "Directory with scan results missing!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -x ./parse_CAs ]; then
|
||||
echo "Compiling parse_CAs script"
|
||||
gcc -o parse_CAs parse_CAs.c -lssl -lcrypto -ljson-c --std=gnu99
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compilation failed, aborting" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Verifying certificate chains from results files"
|
||||
./parse_CAs > parsed
|
||||
echo "Calculating statistics for verified certificate chains"
|
||||
python parse_CAs.py > trust_scan
|
||||
echo "Done!"
|
||||
echo "Results are in \"trust_scan\" file"
|
||||
@ -2,20 +2,44 @@
|
||||
parallel=10
|
||||
max_bg=50
|
||||
absolute_max_bg=100
|
||||
max_load=50
|
||||
max_load_avg=50
|
||||
|
||||
if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then
|
||||
echo "max user processes too low, use ulimit -u to increase"
|
||||
exit 1
|
||||
fi
|
||||
[ ! -e "results" ] && mkdir results
|
||||
[ ! -e "certs" ] && mkdir certs
|
||||
if [ ! -e "ca_files" ]; then
|
||||
mkdir ca_files
|
||||
pushd ca_files >/dev/null
|
||||
awk '
|
||||
split_after == 1 {n++;split_after=0}
|
||||
/-----END CERTIFICATE-----/ {split_after=1}
|
||||
{print > "cert" n ".pem"}' < "/etc/pki/tls/certs/ca-bundle.crt"
|
||||
for i in *; do
|
||||
h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null)
|
||||
for num in `seq 0 100`; do
|
||||
if [[ $h.$num -ef $i ]]; then
|
||||
# file already linked, ignore
|
||||
break
|
||||
fi
|
||||
if [[ ! -e $h.$num ]]; then
|
||||
# file doesn't exist, create a link
|
||||
ln -s "$i" "$h.$num"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
popd >/dev/null
|
||||
fi
|
||||
|
||||
function wait_for_jobs() {
|
||||
local no_jobs
|
||||
no_jobs=$(jobs | wc -l)
|
||||
|
||||
while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do
|
||||
if awk -v maxload=$max_load '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then
|
||||
while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load_avg '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do
|
||||
if awk -v maxload=$max_load_avg '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then
|
||||
return
|
||||
fi
|
||||
sleep 1
|
||||
@ -32,7 +56,7 @@ function scan_host() {
|
||||
if [ $? -gt 0 ]; then
|
||||
return
|
||||
fi
|
||||
../cipherscan -json -servername $1 $2:443 > results/$1@$2
|
||||
../cipherscan --capath ca_files --savecrt certs -json -servername $1 $2:443 > results/$1@$2
|
||||
}
|
||||
|
||||
function scan_host_no_sni() {
|
||||
@ -44,10 +68,12 @@ function scan_host_no_sni() {
|
||||
if [ $? -gt 0 ]; then
|
||||
return
|
||||
fi
|
||||
../cipherscan -json $1:443 > results/$1
|
||||
../cipherscan --capath ca_files --savecrt certs -json $1:443 > results/$1
|
||||
}
|
||||
|
||||
function scan_hostname() {
|
||||
# check if the hostname isn't an IP address (since we can't put IP
|
||||
# addresses to SNI extension)
|
||||
if [[ ! -z $(awk -F. '$1>=0 && $1<=255 && $2>=0 && $2<=255 &&
|
||||
$3>=0 && $3<=255 && $4>=0 && $4<=255 && NF==4' <<<"$1") ]]; then
|
||||
scan_host_no_sni $1
|
||||
|
||||
Loading…
Reference in New Issue
Block a user