LittleNellie
/
cipherscan
mirror of https://github.com/mozilla/cipherscan.git
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #18 from tomato42/wip 

Hodgepodge of fixes
This commit is contained in:
Julien Vehent 2014-08-28 16:02:19 -04:00
parent 273211f025 7591062bbc
commit ecd77f94fc
4 changed files with 1000165 additions and 1000003 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
cipherscan
View File

@ -9,10 +9,13 @@ DOBENCHMARK=0
BENCHMARKITER=30
OPENSSLBIN="$(dirname $0)/openssl"
CACERTS=${CACERTS:-/etc/pki/tls/certs/ca-bundle.crt}
if [ ! -e "$CACERTS" ]; then
CACERTS="/etc/ssl/certs/ca-certificates.crt"
fi
if [ ! -e "$CACERTS" ]; then
echo "Warning: CA Certificates not found at $CACERTS, export CACERTS variable with location of your trust anchors" 1>&2
fi
CIPHERSUITE="ALL:COMPLEMENTOFALL"
CIPHERSUITE="ALL:COMPLEMENTOFALL:+aRSA"
DEBUG=0
VERBOSE=0
DELAY=0
@ -75,6 +78,13 @@ test_cipher_on_target() {
do
debug echo \"Q\" \| $sslcommand $tls_version
local tmp=$(echo "Q" | $sslcommand $tls_version 1>/dev/stdout 2>/dev/null)
if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then
current_ocspstaple="True"
else
current_ocspstaple="False"
fi
# filter out the OCSP server certificate
tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp")
current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}')
current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}')
current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}')
@ -82,7 +92,11 @@ test_cipher_on_target() {
if [ -z $current_pubkey ]; then
current_pubkey=0
fi
current_sigalg=$(openssl x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }')
if [ -z $current_tickethint ]; then
current_tickethint=None
fi
current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null
if [ $? -eq 0 ]; then
current_trusted="True"
@ -115,6 +129,8 @@ test_cipher_on_target() {
pubkey=$current_pubkey
sigalg=$current_sigalg
trusted=$current_trusted
tickethint=$current_tickethint
ocspstaple=$current_ocspstaple
# grab the cipher and PFS key size
done
# if cipher is empty, that means none of the TLS version worked with
@ -126,13 +142,13 @@ test_cipher_on_target() {
# if cipher contains NONE, the cipher wasn't accepted
elif [ "$cipher" == '(NONE) ' ]; then
result="$cipher $protocols $pubkey $sigalg $trusted $pfs"
result="$cipher $protocols $pubkey $sigalg $trusted $tickethint $ocspstaple $pfs"
verbose "handshake failed, server returned ciphersuite '$result'"
return 1
# the connection succeeded
else
result="$cipher $protocols $pubkey $sigalg $trusted $pfs"
result="$cipher $protocols $pubkey $sigalg $trusted $tickethint $ocspstaple $pfs"
verbose "handshake succeeded, server returned ciphersuite '$result'"
return 0
fi
@ -166,9 +182,9 @@ get_cipher_pref() {
[ "$OUTPUTFORMAT" == "terminal" ] && [ $DEBUG -lt 1 ] && echo -n '.'
local ciphersuite="$1"
if [ -e $CACERTS ]; then
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
else
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
fi
verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'"
test_cipher_on_target "$sslcommand"
@ -190,6 +206,8 @@ display_results_in_terminal() {
local pubkey
local sigalg
local trusted
local tickethint
local ocspstaple
local different=False
for cipher in "${cipherspref[@]}"; do
pciph=$(echo $cipher|awk '{print $1}')
@ -203,6 +221,8 @@ display_results_in_terminal() {
pubkey=$(awk '{print $3}' <<<$cipher)
sigalg=$(awk '{print $4}' <<<$cipher)
trusted=$(awk '{print $5}' <<<$cipher)
tickethint=$(awk '{print $6}' <<<$cipher)
ocspstaple=$(awk '{print $7}' <<<$cipher)
else
if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then
different=True
@ -213,6 +233,9 @@ display_results_in_terminal() {
if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then
different=True
fi
if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then
different=True
fi
fi
results=("${results[@]}" "$r")
ctr=$((ctr+1))
@ -220,13 +243,13 @@ display_results_in_terminal() {
if [ $DOBENCHMARK -eq 1 ]; then
if [ $different == "True" ]; then
header="prio ciphersuite protocols pubkey_size signature_algoritm trusted pfs_keysize avg_handshake_microsec"
header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint pfs_keysize avg_handshake_microsec"
else
header="prio ciphersuite protocols pfs_keysize avg_handshake_microsec"
fi
else
if [ $different == "True" ]; then
header="prio ciphersuite protocols pubkey_size signature_algorithm trusted pfs_keysize"
header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint pfs_keysize"
else
header="prio ciphersuite protocols pfs_keysize"
fi
@ -240,7 +263,8 @@ display_results_in_terminal() {
if [ $different == "True" ]; then
echo $result|grep -v '(NONE)'
else
echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $7}'
# prints priority, ciphersuite, protocols and pfs_keysize
echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}'
fi
done|column -t
echo
@ -250,6 +274,12 @@ display_results_in_terminal() {
else
echo "Certificate: UNTRUSTED, $pubkey bit, $sigalg signature"
fi
echo "TLS ticket lifetime hint: $tickethint"
fi
if [[ $ocspstaple == "True" ]]; then
echo "OCSP stapling: supported"
else
echo "OCSP stapling: not supported"
fi
}
@ -265,7 +295,9 @@ display_results_in_json() {
echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"],"
echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"],"
echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\","
pfs=$(echo $cipher|awk '{print $6}')
echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\","
echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\","
pfs=$(echo $cipher|awk '{print $8}')
[ "$pfs" == "" ] && pfs="None"
echo -n "\"pfs\":\"$pfs\"}"
ctr=$((ctr+1))

77
top1m/parse_results.py
View File

@ -23,6 +23,8 @@ protocolstats = defaultdict(int)
handshakestats = defaultdict(int)
keysize = defaultdict(int)
sigalg = defaultdict(int)
tickethint = defaultdict(int)
ocspstaple = defaultdict(int)
dsarsastack = 0
total = 0
for r,d,flist in os.walk(path):
@ -35,6 +37,8 @@ for r,d,flist in os.walk(path):
tempecckeystats = {}
tempdsakeystats = {}
tempsigstats = {}
tempticketstats = {}
tempcipherstats = {}
ciphertypes = 0
AESGCM = False
AES = False
@ -42,9 +46,13 @@ for r,d,flist in os.walk(path):
DES3 = False
CAMELLIA = False
RC4 = False
ADH = False
DHE = False
AECDH = False
ECDHE = False
RSA = False
ECDH = False
DH = False
SSL2 = False
SSL3 = False
TLS1 = False
@ -53,6 +61,7 @@ for r,d,flist in os.walk(path):
dualstack = False
ECDSA = False
trusted = False
ocsp_stapling = None
""" process the file """
f_abs = os.path.join(r,f)
@ -77,7 +86,11 @@ for r,d,flist in os.walk(path):
continue
""" store the ciphers supported """
if 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']:
if 'ADH' in entry['cipher'] or 'AECDH' in entry['cipher']:
ciphertypes += 1
name = "z:" + entry['cipher']
tempcipherstats[name] = 1
elif 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']:
if not AESGCM:
AESGCM = True
ciphertypes += 1
@ -104,15 +117,25 @@ for r,d,flist in os.walk(path):
else:
ciphertypes += 1
name = "z:" + entry['cipher']
cipherstats[name] += 1
tempcipherstats[name] = 1
""" store key handshake methods """
if 'ECDHE' in entry['cipher']:
ECDHE = True
temppfsstats[entry['pfs']] = 1
elif 'DHE' in entry['cipher']:
elif 'DHE' in entry['cipher'] or 'EDH' in entry['cipher']:
DHE = True
temppfsstats[entry['pfs']] = 1
elif 'AECDH' in entry['cipher']:
AECDH = True
elif 'ADH' in entry['cipher']:
ADH = True
elif 'ECDH' in entry['cipher']:
ECDH = True
elif 'DH' in entry['cipher']:
DH = True
else:
RSA = True
""" save the key size """
if 'ECDSA' in entry['cipher']:
@ -133,6 +156,17 @@ for r,d,flist in os.walk(path):
""" save key signatures size """
tempsigstats[entry['sigalg'][0]] = 1
""" save tls ticket hint """
if 'ticket_hint' in entry:
tempticketstats[entry['ticket_hint']] = 1
""" check if OCSP stapling is supported """
if 'ocsp_stapling' in entry:
if entry['ocsp_stapling'] == 'True':
ocsp_stapling=True
else:
ocsp_stapling=False
""" store the versions of TLS supported """
for protocol in entry['protocols']:
if protocol == 'SSLv2':
@ -175,6 +209,19 @@ for r,d,flist in os.walk(path):
for s in tempsigstats:
sigalg[s] += 1
if len(tempticketstats) == 1:
for s in tempticketstats:
tickethint[s + " only"] += 1
for s in tempticketstats:
tickethint[s] += 1
if ocsp_stapling is None:
ocspstaple['Unknown'] += 1
elif ocsp_stapling:
ocspstaple['Supported'] += 1
else:
ocspstaple['Unsupported'] += 1
""" store cipher stats """
if AESGCM:
cipherstats['AES-GCM'] += 1
@ -209,12 +256,24 @@ for r,d,flist in os.walk(path):
cipherstats['RC4 forced in TLS1.1+'] += 1
cipherstats['RC4 Preferred'] += 1
for cipher in tempcipherstats:
cipherstats[cipher] += 1
""" store handshake stats """
if AECDH:
handshakestats['AECDH'] += 1
if ADH:
handshakestats['ADH'] += 1
if ECDHE:
handshakestats['ECDHE'] += 1
if DHE:
handshakestats['DHE'] += 1
if DHE and ECDHE:
handshakestats['ECDHE and DHE'] += 1
if ECDH:
handshakestats['ECDH'] += 1
if DH:
handshakestats['DH'] += 1
if RSA:
handshakestats['RSA'] += 1
@ -279,6 +338,12 @@ for stat in sorted(pfsstats):
pfspercent = round(pfsstats[stat] / handshakestats['DHE'] * 100, 4)
sys.stdout.write(stat.ljust(25) + " " + str(pfsstats[stat]).ljust(10) + str(percent).ljust(9) + str(pfspercent) + "\n")
print("\nTLS session ticket hint Count Percent ")
print("-------------------------+---------+--------")
for stat in sorted(tickethint):
percent = round(tickethint[stat] / total * 100, 4)
sys.stdout.write(stat.ljust(25) + " " + str(tickethint[stat]).ljust(10) + str(percent).ljust(9) + "\n")
print("\nCertificate sig alg Count Percent ")
print("-------------------------+---------+--------")
for stat in sorted(sigalg):
@ -293,6 +358,12 @@ for stat in sorted(keysize):
sys.stdout.write("RSA/ECDSA Dual Stack".ljust(25) + " " + str(dsarsastack).ljust(10) + str(round(dsarsastack/total * 100, 4)) + "\n")
print("\nOCSP stapling Count Percent ")
print("-------------------------+---------+--------")
for stat in sorted(ocspstaple):
percent = round(ocspstaple[stat] / total * 100, 4)
sys.stdout.write(stat.ljust(25) + " " + str(ocspstaple[stat]).ljust(10) + str(percent).ljust(9) + "\n")
print("\nSupported Protocols Count Percent")
print("-------------------------+---------+-------")
for stat in sorted(protocolstats):

87
top1m/testtop1m.sh
View File

@ -1,35 +1,94 @@
#!/usr/bin/env bash
parallel=50
max_bg=400
parallel=10
max_bg=50
absolute_max_bg=100
max_load=50
if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then
echo "max user processes too low, use ulimit -u to increase"
exit 1
fi
[ ! -e "results" ] && mkdir results
function wait_for_jobs() {
local no_jobs
no_jobs=$(jobs | wc -l)
while [ $no_jobs -gt $1 ]; do
while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do
if awk -v maxload=$max_load '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then
return
fi
sleep 1
no_jobs=$(jobs | wc -l)
done
}
function scan_host() {
# do not scan the same host multiple times
if [ -e results/$1@$2 ]; then
return
fi
tcping -u 10000000 $2 443;
if [ $? -gt 0 ]; then
return
fi
../cipherscan -json -servername $1 $2:443 > results/$1@$2
}
function scan_host_no_sni() {
# do not scan the same host multiple times
if [ -e results/$1 ]; then
return
fi
tcping -u 10000000 $1 443;
if [ $? -gt 0 ]; then
return
fi
../cipherscan -json $1:443 > results/$1
}
function scan_hostname() {
if [[ ! -z $(awk -F. '$1>=0 && $1<=255 && $2>=0 && $2<=255 &&
$3>=0 && $3<=255 && $4>=0 && $4<=255 && NF==4' <<<"$1") ]]; then
scan_host_no_sni $1
return
fi
local host_ips=$(host $1 | awk '/has address/ {print $4}')
local www_ips=$(host www.$1 | awk '/has address/ {print $4}')
if [ ! -z "$host_ips" ] && [ ! -z "$www_ips" ]; then
# list of IPs that are in www but not in host
local diff=$(grep -Fv "$host_ips" <<< "$www_ips")
head -n 1 <<< "$host_ips" | while read ip; do
scan_host $1 $ip
done
if [ ! -z "$diff" ]; then
head -n 1 <<<"$diff" | while read ip; do
scan_host www.$1 $ip
done
fi
else
if [ ! -z "$host_ips" ]; then
head -n 1 <<<"$host_ips" | while read ip; do
scan_host $1 $ip
done
fi
if [ ! -z "$www_ips" ]; then
head -n 1 <<<"$www_ips" | while read ip; do
scan_host www.$1 $ip
done
fi
fi
}
i=0
count=$(wc -l top-1m.csv | awk '{print $1}')
while [ $i -lt $count ]
do
echo processings sites $i to $((i + parallel))
for t in $(tail -$(($count - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2)
for t in $(tail -$(($count - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2|cut -d "/" -f 1)
do
(tcping -u 10000000 $t 443;
if [ $? -gt 0 ];then
tcping -u 10000000 www.$t 443;
if [ $? -gt 0 ]; then
continue;
else
../cipherscan -json www.$t:443 > results/www.$t
continue;
fi;
fi;../cipherscan -json $t:443 > results/$t )&
(scan_hostname $t)&
done
i=$(( i + parallel))
wait_for_jobs $max_bg

1999952
top1m/top-1m.csv
View File

File diff suppressed because it is too large Load Diff