diff --git a/cipherscan b/cipherscan index 3990ea6..6efb49c 100755 --- a/cipherscan +++ b/cipherscan @@ -47,6 +47,11 @@ TIMEOUT=30 # place where to put the found intermediate CA certificates and where # trust anchors are stored CAPATH="" +SAVECRT="" +unset known_certs +declare -A known_certs +unset cert_checksums +declare -A cert_checksums # because running external commands like sleep incurs a fork penalty, we # first check if it is necessary @@ -57,7 +62,9 @@ ratelimit() { } usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] +[--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] +[-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -71,12 +78,14 @@ Use one of the options below: -a | --allciphers Test all known ciphers individually at the end. -b | --benchmark Activate benchmark mode. ---capath use CAs from directory +--capath use CAs from directory (must be in OpenSSL CAdir format) +--saveca save intermediate certificates in CA directory -d | --delay Pause for n seconds between connections -D | --debug Output ALL the information. -h | --help Shows this help text. -j | --json Output results in JSON format. -o | --openssl path/to/your/openssl binary you want to use. +--savecrt path where to save untrusted and leaf certificates -v | --verbose Increase verbosity. The rest of the arguments will be interpreted as openssl s_client argument. @@ -99,6 +108,123 @@ debug(){ fi } +c_hash() { + local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) + for ((num=0; num<=100; num++)) ; do + if [[ $1/${h}.${num} -ef $2 ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $1/${h}.${num} ]]; then + # file doesn't exist, create a link + pushd "$1" > /dev/null + ln -s "$2" "${h}.${num}" + popd > /dev/null + break + fi + done +} + +parse_openssl_output() { + # clear variables in case matching doesn't hit them + current_ocspstaple="False" + current_cipher="" + current_pfs="" + current_protocol="" + current_tickethint="None" + current_pubkey=0 + current_trusted="False" + current_sigalg="None" + + certs_found=0 + current_raw_certificates=() + + while read line; do + # check if there isn't OCSP response data (response and responder cert) + if [[ $line =~ ^====================================== ]]; then + while read data; do + # check if there is a OCSP response in output + if [[ $data =~ OCSP\ Response\ Data ]]; then + current_ocspstaple="True" + continue + fi + + # skip all data from a OCSP response + if [[ $data =~ ^====================================== ]]; then + break + fi + done + continue + fi + + # extract selected cipher + if [[ $line =~ New,\ ]]; then + local match=($line) + current_cipher="${match[4]}" + continue + fi + + # extract data about selected temporary key + if [[ $line =~ Server\ Temp\ Key ]]; then + local match=($line) + current_pfs="${match[3]}${match[4]}${match[5]}${match[6]}" + continue + fi + + # extract used protocol + if [[ $line =~ ^Protocol\ + ]]; then + local match=($line) + current_protocol="${match[2]}" + continue + fi + + # extract session ticket hint + if [[ $line =~ ticket\ lifetime\ hint ]]; then + local match=($line) + current_tickethint="${match[5]}" + continue + fi + + # extract size of server public key + if [[ $line =~ Server\ public\ key\ is\ ]]; then + local match=($line) + current_pubkey="${match[4]}" + continue + fi + + # check if connection used trused certificate + if [[ $line =~ Verify\ return\ code:\ 0 ]]; then + current_trusted="True" + continue + fi + + # extract certificates + if [[ $line =~ -----BEGIN\ CERTIFICATE----- ]]; then + current_raw_certificates[$certs_found]="$line"$'\n' + while read data; do + current_raw_certificates[$certs_found]+="$data"$'\n' + if [[ $data =~ -----END\ CERTIFICATE----- ]]; then + break + fi + done + certs_found=$((certs_found+1)) + continue + fi + done + + # if we found any certs in output, process the first one and extract + # the signature algorithm on it (it's the server's certificate) + if [[ $certs_found -gt 0 ]]; then + local ossl_out=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"${current_raw_certificates[0]}") + while read data; do + if [[ $data =~ Signature\ Algorithm ]]; then + local match=($data) + current_sigalg="${match[2]}" + fi + done <<<"$ossl_out" + fi +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -107,51 +233,102 @@ test_cipher_on_target() { protocols="" pfs="" previous_cipher="" + certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do # sslv2 client hello doesn't support SNI extension # in SSLv3 mode OpenSSL just ignores the setting so it's ok # -status exception is ignored in SSLv2, go figure if [ "$tls_version" == "-ssl2" ]; then - cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand) + if [[ "$sslcommand" =~ (.*)(-servername\ [^ ]*)(.*) ]]; then + cmnd="${BASH_REMATCH[1]} ${BASH_REMATCH[3]}" + else + cmnd="$sslcommand" + fi else cmnd=$sslcommand fi ratelimit debug echo \"Q\" \| $cmnd $tls_version local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null) - if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then - current_ocspstaple="True" - else - current_ocspstaple="False" - fi - # filter out the OCSP server certificate - tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") - # session metadata - current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') - current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') - current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') - current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }') - if [ -z $current_tickethint ]; then - current_tickethint=None - fi + parse_openssl_output <<<"$tmp" + verbose "selected cipher is '$current_cipher'" + verbose "using protocol '$current_protocol'" - # certificate metadata - current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') - if [ -z $current_pubkey ]; then - current_pubkey=0 - fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" - grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null - if [ $? -eq 0 ]; then - current_trusted="True" - else - current_trusted="False" - fi - if [ -z $current_sigalg ]; then - current_sigalg=None - fi + # collect certificate data + current_certificates="" + local certificate_count=$certs_found + debug "server presented $certificate_count certificates" + local i + for ((i=0; i<$certificate_count; i=i+1 )); do + + # extract i'th certificate + local cert="${current_raw_certificates[$i]}" + # put the output to an array instead running awk '{print $1}' + local cksum=($(cksum <<<"$cert")) + # compare the values not just checksums so that eventual collision + # doesn't mess up results + if [[ ${known_certs[$cksum]} == $cert ]]; then + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${cert_checksums[$cksum]}\"" + continue + fi + + # compute sha256 fingerprint of the certificate + local sha256sum=($(${OPENSSLBIN} x509 -outform DER\ + <<<"$cert" 2>/dev/null |\ + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null)) + + # check if it is a CA certificate + local isCA="False" + if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\ + grep 'CA:TRUE' >/dev/null; then + isCA="True" + fi + + # build trust source for certificate verification + local trust_source=() + if [[ -n $CAPATH ]]; then + trust_source=("-CApath" "$CAPATH") + elif [[ -e $CACERTS ]]; then + trust_source=("-CAfile" "$CACERTS") + fi + + # check if the certificate is actually trusted (server may present + # unrelated certificates that are not trusted (including self + # signed ones) + local saved="False" + if ${OPENSSLBIN} verify "${trust_source[@]}" \ + -untrusted <(printf "%s" "${current_raw_certificates[@]}") <(echo "$cert") 2>/dev/null | \ + grep ': OK$' >/dev/null; then + + # if the certificate is an intermediate CA it may be useful + # for connecting to servers that are misconfigured so save it + if [[ -n $CAPATH ]] && [[ $SAVECA == "True" ]] && [[ $isCA == "True" ]]; then + if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then + echo "$cert" > "$CAPATH/${sha256sum}.pem" + c_hash "$CAPATH" "${sha256sum}.pem" + fi + saved="True" + fi + fi + if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then + if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then + echo "$cert" > "$SAVECRT/${sha256sum}.pem" + fi + fi + # save the sha sum for reporting + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${sha256sum}\"" + known_certs[$cksum]="$cert" + cert_checksums[$cksum]="$sha256sum" + done + debug "current_certificates: $current_certificates" # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then @@ -179,6 +356,7 @@ test_cipher_on_target() { trusted=$current_trusted tickethint=$current_tickethint ocspstaple=$current_ocspstaple + certificates="$current_certificates" # grab the cipher and PFS key size done # if cipher is empty, that means none of the TLS version worked with @@ -230,7 +408,7 @@ get_cipher_pref() { local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e $CACERTS ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -242,7 +420,8 @@ get_cipher_pref() { # If the connection succeeded with the current cipher, benchmark and store if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") - pciph=$(echo $result|awk '{print $1}') + ciphercertificates=("${ciphercertificates[@]}" "$certificates") + pciph=($result) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -259,30 +438,32 @@ display_results_in_terminal() { local different=False echo "Target: $TARGET"; echo for cipher in "${cipherspref[@]}"; do - pciph=$(echo $cipher|awk '{print $1}') + # get first in array + pciph=($cipher) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" else r="$ctr $cipher" fi + local cipher_data=($cipher) if [ $ctr -eq 1 ]; then - pubkey=$(awk '{print $3}' <<<$cipher) - sigalg=$(awk '{print $4}' <<<$cipher) - trusted=$(awk '{print $5}' <<<$cipher) - tickethint=$(awk '{print $6}' <<<$cipher) - ocspstaple=$(awk '{print $7}' <<<$cipher) + pubkey="${cipher_data[2]}" + sigalg="${cipher_data[3]}" + trusted="${cipher_data[4]}" + tickethint="${cipher_data[5]}" + ocspstaple="${cipher_data[6]}" else - if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then + if [ "$pubkey" != "${cipher_data[2]}" ]; then different=True fi - if [ "$sigalg" != "$(awk '{print $4}' <<<$cipher)" ]; then + if [ "$sigalg" != "${cipher_data[3]}" ]; then different=True fi - if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then + if [ "$trusted" != "${cipher_data[4]}" ]; then different=True fi - if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then + if [ "$tickethint" != "${cipher_data[5]}" ]; then different=True fi fi @@ -313,7 +494,7 @@ display_results_in_terminal() { echo $result|grep -v '(NONE)' else # prints priority, ciphersuite, protocols and pfs_keysize - echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}' + awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result" fi done|column -t echo @@ -342,15 +523,19 @@ display_results_in_json() { ctr=0 echo -n "{\"target\":\"$TARGET\",\"utctimestamp\":\"$(date -u '+%FT%T.0Z')\",\"serverside\":\"${serverside}\",\"ciphersuite\": [" for cipher in "${cipherspref[@]}"; do + local cipher_arr=($cipher) [ $ctr -gt 0 ] && echo -n ',' - echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\"," - echo -n "\"protocols\":[\"$(echo $cipher|awk '{print $2}'|sed 's/,/","/g')\"]," - echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," - echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," - echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," - echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," - echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," - pfs=$(echo $cipher|awk '{print $8}') + echo -n "{\"cipher\":\"${cipher_arr[0]}\"," + echo -n "\"protocols\":[\"${cipher_arr[1]//,/\",\"}\"]," + echo -n "\"pubkey\":[\"${cipher_arr[2]//,/\",\"}\"]," + echo -n "\"sigalg\":[\"${cipher_arr[3]//,/\",\"}\"]," + echo -n "\"trusted\":\"${cipher_arr[4]//,/\",\"}\"," + if [[ -n $CAPATH ]]; then + echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," + fi + echo -n "\"ticket_hint\":\"${cipher_arr[5]}\"," + echo -n "\"ocsp_stapling\":\"${cipher_arr[6]}\"," + pfs="${cipher_arr[7]}" [ "$pfs" == "" ] && pfs="None" echo -n "\"pfs\":\"$pfs\"}" ctr=$((ctr+1)) @@ -368,31 +553,31 @@ test_serverside_ordering() { # server supports just two ciphers, so rotate them, that should be enough elif [[ ${#cipherspref[@]} -eq 2 ]]; then - local cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + local cipher=(${cipherspref[1]}) prefered="$cipher" ciphersuite=$cipher - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" # server supports 3 or more ciphers, rotate all three. This is necessary because google does # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or # ECDHE-RSA-CHACHA20-POLY1305 else - local cipher=$(awk '{print $1}' <<< ${cipherspref[2]}) + local cipher=(${cipherspref[2]}) prefered="$cipher" ciphersuite="$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + cipher=(${cipherspref[1]}) ciphersuite+=":$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" fi local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e "$CACERTS" ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -402,7 +587,7 @@ test_serverside_ordering() { if [ $? -ne 0 ]; then serverside="True" else - local selected=$(awk '{print $1}' <<< $result) + local selected=($result) if [[ $selected == $prefered ]]; then serverside="False" else @@ -452,6 +637,14 @@ do CAPATH="$2" shift 2 ;; + --saveca) + SAVECA="True" + shift 1 + ;; + --savecrt) + SAVECRT="$2" + shift 2 + ;; --) # End of all options shift break @@ -491,8 +684,7 @@ TARGET=$HOST:$PORT debug "target: $TARGET" # test our openssl is usable -tmp="$($OPENSSLBIN -h 2>&1 1>/dev/null)" -if [ $? -gt 0 ]; then +if [ ! -x $OPENSSLBIN ]; then OPENSSLBIN=$(which openssl) if [ "$OUTPUTFORMAT" == "terminal" ]; then echo "custom openssl not executable, falling back to system one from $OPENSSLBIN" @@ -504,6 +696,7 @@ debug "sclientargs: $SCLIENTARGS" cipherspref=(); +ciphercertificates=() results=() # Call to the recursive loop that retrieves the cipher preferences diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index e3e6920..8a39055 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -9,6 +9,42 @@ if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then exit 1 fi [ ! -e "results" ] && mkdir results +[ ! -e "certs" ] && mkdir certs +if [ -z "$CACERTS" ]; then + for f in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do + if [ -e "$f" ]; then + CACERTS="$f" + break + fi + done +fi +if [ ! -e "$CACERTS" ]; then + echo "file with CA certificates does not exist, please export CACERTS variable with location" + exit 1 +fi +if [ ! -e "ca_files" ]; then + mkdir ca_files + pushd ca_files >/dev/null + awk ' + split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {print > "cert" n ".pem"}' < "$CACERTS" + for i in *; do + h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null) + for num in `seq 0 100`; do + if [[ $h.$num -ef $i ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $h.$num ]]; then + # file doesn't exist, create a link + ln -s "$i" "$h.$num" + break + fi + done + done + popd >/dev/null +fi function wait_for_jobs() { local no_jobs @@ -32,7 +68,7 @@ function scan_host() { if [ $? -gt 0 ]; then return fi - ../cipherscan --delay 2 -json -servername $1 $2:443 > results/$1@$2 + ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json -servername $1 $2:443 > results/$1@$2 } function scan_host_no_sni() { @@ -44,7 +80,7 @@ function scan_host_no_sni() { if [ $? -gt 0 ]; then return fi - ../cipherscan --delay 2 -json $1:443 > results/$1 + ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json $1:443 > results/$1 } function scan_hostname() {