2
0
mirror of https://github.com/mozilla/cipherscan.git synced 2024-12-26 04:33:42 +01:00

Merge pull request #33 from tomato42/cipherscan-fixes-3

Cipherscan fixes, speedups and saving of certificates (v3)
This commit is contained in:
Julien Vehent 2014-11-05 12:36:33 -05:00
commit 818bf29b02
2 changed files with 294 additions and 65 deletions

View File

@ -47,6 +47,11 @@ TIMEOUT=30
# place where to put the found intermediate CA certificates and where
# trust anchors are stored
CAPATH=""
SAVECRT=""
unset known_certs
declare -A known_certs
unset cert_checksums
declare -A cert_checksums
# because running external commands like sleep incurs a fork penalty, we
# first check if it is necessary
@ -57,7 +62,9 @@ ratelimit() {
}
usage() {
echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory]
[--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json]
[-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
usage: $0 -h|--help
$0 attempts to connect to a target site using all the ciphersuites it knows.
@ -71,12 +78,14 @@ Use one of the options below:
-a | --allciphers Test all known ciphers individually at the end.
-b | --benchmark Activate benchmark mode.
--capath use CAs from directory
--capath use CAs from directory (must be in OpenSSL CAdir format)
--saveca save intermediate certificates in CA directory
-d | --delay Pause for n seconds between connections
-D | --debug Output ALL the information.
-h | --help Shows this help text.
-j | --json Output results in JSON format.
-o | --openssl path/to/your/openssl binary you want to use.
--savecrt path where to save untrusted and leaf certificates
-v | --verbose Increase verbosity.
The rest of the arguments will be interpreted as openssl s_client argument.
@ -99,6 +108,123 @@ debug(){
fi
}
c_hash() {
local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null)
for ((num=0; num<=100; num++)) ; do
if [[ $1/${h}.${num} -ef $2 ]]; then
# file already linked, ignore
break
fi
if [[ ! -e $1/${h}.${num} ]]; then
# file doesn't exist, create a link
pushd "$1" > /dev/null
ln -s "$2" "${h}.${num}"
popd > /dev/null
break
fi
done
}
parse_openssl_output() {
# clear variables in case matching doesn't hit them
current_ocspstaple="False"
current_cipher=""
current_pfs=""
current_protocol=""
current_tickethint="None"
current_pubkey=0
current_trusted="False"
current_sigalg="None"
certs_found=0
current_raw_certificates=()
while read line; do
# check if there isn't OCSP response data (response and responder cert)
if [[ $line =~ ^====================================== ]]; then
while read data; do
# check if there is a OCSP response in output
if [[ $data =~ OCSP\ Response\ Data ]]; then
current_ocspstaple="True"
continue
fi
# skip all data from a OCSP response
if [[ $data =~ ^====================================== ]]; then
break
fi
done
continue
fi
# extract selected cipher
if [[ $line =~ New,\ ]]; then
local match=($line)
current_cipher="${match[4]}"
continue
fi
# extract data about selected temporary key
if [[ $line =~ Server\ Temp\ Key ]]; then
local match=($line)
current_pfs="${match[3]}${match[4]}${match[5]}${match[6]}"
continue
fi
# extract used protocol
if [[ $line =~ ^Protocol\ + ]]; then
local match=($line)
current_protocol="${match[2]}"
continue
fi
# extract session ticket hint
if [[ $line =~ ticket\ lifetime\ hint ]]; then
local match=($line)
current_tickethint="${match[5]}"
continue
fi
# extract size of server public key
if [[ $line =~ Server\ public\ key\ is\ ]]; then
local match=($line)
current_pubkey="${match[4]}"
continue
fi
# check if connection used trused certificate
if [[ $line =~ Verify\ return\ code:\ 0 ]]; then
current_trusted="True"
continue
fi
# extract certificates
if [[ $line =~ -----BEGIN\ CERTIFICATE----- ]]; then
current_raw_certificates[$certs_found]="$line"$'\n'
while read data; do
current_raw_certificates[$certs_found]+="$data"$'\n'
if [[ $data =~ -----END\ CERTIFICATE----- ]]; then
break
fi
done
certs_found=$((certs_found+1))
continue
fi
done
# if we found any certs in output, process the first one and extract
# the signature algorithm on it (it's the server's certificate)
if [[ $certs_found -gt 0 ]]; then
local ossl_out=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"${current_raw_certificates[0]}")
while read data; do
if [[ $data =~ Signature\ Algorithm ]]; then
local match=($data)
current_sigalg="${match[2]}"
fi
done <<<"$ossl_out"
fi
}
# Connect to a target host with the selected ciphersuite
test_cipher_on_target() {
local sslcommand=$@
@ -107,51 +233,102 @@ test_cipher_on_target() {
protocols=""
pfs=""
previous_cipher=""
certificates=""
for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2"
do
# sslv2 client hello doesn't support SNI extension
# in SSLv3 mode OpenSSL just ignores the setting so it's ok
# -status exception is ignored in SSLv2, go figure
if [ "$tls_version" == "-ssl2" ]; then
cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand)
if [[ "$sslcommand" =~ (.*)(-servername\ [^ ]*)(.*) ]]; then
cmnd="${BASH_REMATCH[1]} ${BASH_REMATCH[3]}"
else
cmnd="$sslcommand"
fi
else
cmnd=$sslcommand
fi
ratelimit
debug echo \"Q\" \| $cmnd $tls_version
local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null)
if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then
current_ocspstaple="True"
else
current_ocspstaple="False"
fi
# filter out the OCSP server certificate
tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp")
# session metadata
current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}')
current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}')
current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}')
current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }')
if [ -z $current_tickethint ]; then
current_tickethint=None
fi
parse_openssl_output <<<"$tmp"
verbose "selected cipher is '$current_cipher'"
verbose "using protocol '$current_protocol'"
# certificate metadata
current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}')
if [ -z $current_pubkey ]; then
current_pubkey=0
fi
current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null
if [ $? -eq 0 ]; then
current_trusted="True"
else
current_trusted="False"
fi
if [ -z $current_sigalg ]; then
current_sigalg=None
fi
# collect certificate data
current_certificates=""
local certificate_count=$certs_found
debug "server presented $certificate_count certificates"
local i
for ((i=0; i<$certificate_count; i=i+1 )); do
# extract i'th certificate
local cert="${current_raw_certificates[$i]}"
# put the output to an array instead running awk '{print $1}'
local cksum=($(cksum <<<"$cert"))
# compare the values not just checksums so that eventual collision
# doesn't mess up results
if [[ ${known_certs[$cksum]} == $cert ]]; then
if [ -n "${current_certificates}" ]; then
current_certificates+=","
fi
current_certificates+="\"${cert_checksums[$cksum]}\""
continue
fi
# compute sha256 fingerprint of the certificate
local sha256sum=($(${OPENSSLBIN} x509 -outform DER\
<<<"$cert" 2>/dev/null |\
${OPENSSLBIN} dgst -sha256 -r 2>/dev/null))
# check if it is a CA certificate
local isCA="False"
if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\
grep 'CA:TRUE' >/dev/null; then
isCA="True"
fi
# build trust source for certificate verification
local trust_source=()
if [[ -n $CAPATH ]]; then
trust_source=("-CApath" "$CAPATH")
elif [[ -e $CACERTS ]]; then
trust_source=("-CAfile" "$CACERTS")
fi
# check if the certificate is actually trusted (server may present
# unrelated certificates that are not trusted (including self
# signed ones)
local saved="False"
if ${OPENSSLBIN} verify "${trust_source[@]}" \
-untrusted <(printf "%s" "${current_raw_certificates[@]}") <(echo "$cert") 2>/dev/null | \
grep ': OK$' >/dev/null; then
# if the certificate is an intermediate CA it may be useful
# for connecting to servers that are misconfigured so save it
if [[ -n $CAPATH ]] && [[ $SAVECA == "True" ]] && [[ $isCA == "True" ]]; then
if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then
echo "$cert" > "$CAPATH/${sha256sum}.pem"
c_hash "$CAPATH" "${sha256sum}.pem"
fi
saved="True"
fi
fi
if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then
if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then
echo "$cert" > "$SAVECRT/${sha256sum}.pem"
fi
fi
# save the sha sum for reporting
if [ -n "${current_certificates}" ]; then
current_certificates+=","
fi
current_certificates+="\"${sha256sum}\""
known_certs[$cksum]="$cert"
cert_checksums[$cksum]="$sha256sum"
done
debug "current_certificates: $current_certificates"
# parsing finished, report result
if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then
@ -179,6 +356,7 @@ test_cipher_on_target() {
trusted=$current_trusted
tickethint=$current_tickethint
ocspstaple=$current_ocspstaple
certificates="$current_certificates"
# grab the cipher and PFS key size
done
# if cipher is empty, that means none of the TLS version worked with
@ -230,7 +408,7 @@ get_cipher_pref() {
local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client"
if [ -n "$CAPATH" ]; then
sslcommand+=" -CApath $CAPATH"
sslcommand+=" -CApath $CAPATH -showcerts"
elif [ -e $CACERTS ]; then
sslcommand+=" -CAfile $CACERTS"
fi
@ -242,7 +420,8 @@ get_cipher_pref() {
# If the connection succeeded with the current cipher, benchmark and store
if [ $success -eq 0 ]; then
cipherspref=("${cipherspref[@]}" "$result")
pciph=$(echo $result|awk '{print $1}')
ciphercertificates=("${ciphercertificates[@]}" "$certificates")
pciph=($result)
get_cipher_pref "!$pciph:$ciphersuite"
return 0
fi
@ -259,30 +438,32 @@ display_results_in_terminal() {
local different=False
echo "Target: $TARGET"; echo
for cipher in "${cipherspref[@]}"; do
pciph=$(echo $cipher|awk '{print $1}')
# get first in array
pciph=($cipher)
if [ $DOBENCHMARK -eq 1 ]; then
bench_cipher "$pciph"
r="$ctr $cipher $cipherbenchms"
else
r="$ctr $cipher"
fi
local cipher_data=($cipher)
if [ $ctr -eq 1 ]; then
pubkey=$(awk '{print $3}' <<<$cipher)
sigalg=$(awk '{print $4}' <<<$cipher)
trusted=$(awk '{print $5}' <<<$cipher)
tickethint=$(awk '{print $6}' <<<$cipher)
ocspstaple=$(awk '{print $7}' <<<$cipher)
pubkey="${cipher_data[2]}"
sigalg="${cipher_data[3]}"
trusted="${cipher_data[4]}"
tickethint="${cipher_data[5]}"
ocspstaple="${cipher_data[6]}"
else
if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then
if [ "$pubkey" != "${cipher_data[2]}" ]; then
different=True
fi
if [ "$sigalg" != "$(awk '{print $4}' <<<$cipher)" ]; then
if [ "$sigalg" != "${cipher_data[3]}" ]; then
different=True
fi
if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then
if [ "$trusted" != "${cipher_data[4]}" ]; then
different=True
fi
if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then
if [ "$tickethint" != "${cipher_data[5]}" ]; then
different=True
fi
fi
@ -313,7 +494,7 @@ display_results_in_terminal() {
echo $result|grep -v '(NONE)'
else
# prints priority, ciphersuite, protocols and pfs_keysize
echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}'
awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result"
fi
done|column -t
echo
@ -342,15 +523,19 @@ display_results_in_json() {
ctr=0
echo -n "{\"target\":\"$TARGET\",\"utctimestamp\":\"$(date -u '+%FT%T.0Z')\",\"serverside\":\"${serverside}\",\"ciphersuite\": ["
for cipher in "${cipherspref[@]}"; do
local cipher_arr=($cipher)
[ $ctr -gt 0 ] && echo -n ','
echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\","
echo -n "\"protocols\":[\"$(echo $cipher|awk '{print $2}'|sed 's/,/","/g')\"],"
echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"],"
echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"],"
echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\","
echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\","
echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\","
pfs=$(echo $cipher|awk '{print $8}')
echo -n "{\"cipher\":\"${cipher_arr[0]}\","
echo -n "\"protocols\":[\"${cipher_arr[1]//,/\",\"}\"],"
echo -n "\"pubkey\":[\"${cipher_arr[2]//,/\",\"}\"],"
echo -n "\"sigalg\":[\"${cipher_arr[3]//,/\",\"}\"],"
echo -n "\"trusted\":\"${cipher_arr[4]//,/\",\"}\","
if [[ -n $CAPATH ]]; then
echo -n "\"certificates\":[${ciphercertificates[$ctr]}],"
fi
echo -n "\"ticket_hint\":\"${cipher_arr[5]}\","
echo -n "\"ocsp_stapling\":\"${cipher_arr[6]}\","
pfs="${cipher_arr[7]}"
[ "$pfs" == "" ] && pfs="None"
echo -n "\"pfs\":\"$pfs\"}"
ctr=$((ctr+1))
@ -368,31 +553,31 @@ test_serverside_ordering() {
# server supports just two ciphers, so rotate them, that should be enough
elif [[ ${#cipherspref[@]} -eq 2 ]]; then
local cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
local cipher=(${cipherspref[1]})
prefered="$cipher"
ciphersuite=$cipher
cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
cipher=(${cipherspref[0]})
ciphersuite+=":$cipher"
# server supports 3 or more ciphers, rotate all three. This is necessary because google does
# select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or
# ECDHE-RSA-CHACHA20-POLY1305
else
local cipher=$(awk '{print $1}' <<< ${cipherspref[2]})
local cipher=(${cipherspref[2]})
prefered="$cipher"
ciphersuite="$cipher"
cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
cipher=(${cipherspref[1]})
ciphersuite+=":$cipher"
cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
cipher=(${cipherspref[0]})
ciphersuite+=":$cipher"
fi
local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client"
if [ -n "$CAPATH" ]; then
sslcommand+=" -CApath $CAPATH"
sslcommand+=" -CApath $CAPATH -showcerts"
elif [ -e "$CACERTS" ]; then
sslcommand+=" -CAfile $CACERTS"
fi
@ -402,7 +587,7 @@ test_serverside_ordering() {
if [ $? -ne 0 ]; then
serverside="True"
else
local selected=$(awk '{print $1}' <<< $result)
local selected=($result)
if [[ $selected == $prefered ]]; then
serverside="False"
else
@ -452,6 +637,14 @@ do
CAPATH="$2"
shift 2
;;
--saveca)
SAVECA="True"
shift 1
;;
--savecrt)
SAVECRT="$2"
shift 2
;;
--) # End of all options
shift
break
@ -491,8 +684,7 @@ TARGET=$HOST:$PORT
debug "target: $TARGET"
# test our openssl is usable
tmp="$($OPENSSLBIN -h 2>&1 1>/dev/null)"
if [ $? -gt 0 ]; then
if [ ! -x $OPENSSLBIN ]; then
OPENSSLBIN=$(which openssl)
if [ "$OUTPUTFORMAT" == "terminal" ]; then
echo "custom openssl not executable, falling back to system one from $OPENSSLBIN"
@ -504,6 +696,7 @@ debug "sclientargs: $SCLIENTARGS"
cipherspref=();
ciphercertificates=()
results=()
# Call to the recursive loop that retrieves the cipher preferences

View File

@ -9,6 +9,42 @@ if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then
exit 1
fi
[ ! -e "results" ] && mkdir results
[ ! -e "certs" ] && mkdir certs
if [ -z "$CACERTS" ]; then
for f in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do
if [ -e "$f" ]; then
CACERTS="$f"
break
fi
done
fi
if [ ! -e "$CACERTS" ]; then
echo "file with CA certificates does not exist, please export CACERTS variable with location"
exit 1
fi
if [ ! -e "ca_files" ]; then
mkdir ca_files
pushd ca_files >/dev/null
awk '
split_after == 1 {n++;split_after=0}
/-----END CERTIFICATE-----/ {split_after=1}
{print > "cert" n ".pem"}' < "$CACERTS"
for i in *; do
h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null)
for num in `seq 0 100`; do
if [[ $h.$num -ef $i ]]; then
# file already linked, ignore
break
fi
if [[ ! -e $h.$num ]]; then
# file doesn't exist, create a link
ln -s "$i" "$h.$num"
break
fi
done
done
popd >/dev/null
fi
function wait_for_jobs() {
local no_jobs
@ -32,7 +68,7 @@ function scan_host() {
if [ $? -gt 0 ]; then
return
fi
../cipherscan --delay 2 -json -servername $1 $2:443 > results/$1@$2
../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json -servername $1 $2:443 > results/$1@$2
}
function scan_host_no_sni() {
@ -44,7 +80,7 @@ function scan_host_no_sni() {
if [ $? -gt 0 ]; then
return
fi
../cipherscan --delay 2 -json $1:443 > results/$1
../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json $1:443 > results/$1
}
function scan_hostname() {