From 3b14cd914f3b2acf93a686ab089ea60fe8a0c82c Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 13:42:48 +0200 Subject: [PATCH 1/9] no need to grep the input when we're using awk (v2) awk has an inbuilt version of grep, also truncate processing as soon as we find what we're looking for This version uses slightly different syntax that is compatible with old awk --- cipherscan | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cipherscan b/cipherscan index 3990ea6..c2c1a5b 100755 --- a/cipherscan +++ b/cipherscan @@ -129,20 +129,21 @@ test_cipher_on_target() { tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") # session metadata - current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') - current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') - current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') - current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }') + current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") + current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") + current_protocol=$(awk '/^ +Protocol +:/ {print $3; exit}' <<<"$tmp") + current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") if [ -z $current_tickethint ]; then current_tickethint=None fi # certificate metadata - current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') + current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") if [ -z $current_pubkey ]; then current_pubkey=0 fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" + current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ + awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null if [ $? -eq 0 ]; then current_trusted="True" From 826f7b55413d8da6f6fdfb5bec907aded51669a2 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 15:18:11 +0200 Subject: [PATCH 2/9] add caching of intermediate CA certificates --- cipherscan | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/cipherscan b/cipherscan index c2c1a5b..02831e7 100755 --- a/cipherscan +++ b/cipherscan @@ -57,7 +57,7 @@ ratelimit() { } usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -71,7 +71,8 @@ Use one of the options below: -a | --allciphers Test all known ciphers individually at the end. -b | --benchmark Activate benchmark mode. ---capath use CAs from directory +--capath use CAs from directory (must be in OpenSSL CAdir format) +--saveca save intermediate certificates in CA directory -d | --delay Pause for n seconds between connections -D | --debug Output ALL the information. -h | --help Shows this help text. @@ -99,6 +100,23 @@ debug(){ fi } +c_hash() { + local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) + for num in $(seq 0 100); do + if [[ $1/${h}.${num} -ef $2 ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $1/${h}.${num} ]]; then + # file doesn't exist, create a link + pushd "$1" > /dev/null + ln -s "$2" "${h}.${num}" + popd > /dev/null + break + fi + done +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -107,6 +125,7 @@ test_cipher_on_target() { protocols="" pfs="" previous_cipher="" + certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do # sslv2 client hello doesn't support SNI extension @@ -154,6 +173,63 @@ test_cipher_on_target() { current_sigalg=None fi + # collect certificate data + current_certificates="" + local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ + <<<"$tmp") + debug "server presented $certificate_count certificates" + local i + for ((i=0; i<$certificate_count; i=i+1 )); do + + # extract i'th certificate + local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {if (n == i) print } + ' <<<"$tmp") + + # compute sha256 fingerprint of the certificate + local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + + # check if it is a CA certificate + local isCA="False" + if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\ + grep 'CA:TRUE' >/dev/null; then + isCA="True" + fi + + # build trust source for certificate verification + local trust_source=() + if [[ -n $CAPATH ]]; then + trust_source=("-CApath" "$CAPATH") + elif [[ -e $CACERTS ]]; then + trust_source=("-CAfile" "$CACERTS") + fi + + # check if the certificate is actually trusted (server may present + # unrelated certificates that are not trusted (including self + # signed ones) + if ${OPENSSLBIN} verify "${trust_source[@]}" \ + -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + grep 'OK$' >/dev/null; then + + # if the certificate is an intermediate CA it may be useful + # for connecting to servers that are misconfigured so save it + if [[ -n $CAPATH ]] && [[ $SAVECA == "True" ]] && [[ $isCA == "True" ]]; then + if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then + echo "$cert" > "$CAPATH/${sha256sum}.pem" + c_hash "$CAPATH" "${sha256sum}.pem" + fi + fi + fi + # save the sha sum for reporting + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${sha256sum}\"" + done + debug "current_certificates: $current_certificates" + # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then # connection failed, try again with next TLS version @@ -180,6 +256,7 @@ test_cipher_on_target() { trusted=$current_trusted tickethint=$current_tickethint ocspstaple=$current_ocspstaple + certificates="$current_certificates" # grab the cipher and PFS key size done # if cipher is empty, that means none of the TLS version worked with @@ -231,7 +308,7 @@ get_cipher_pref() { local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e $CACERTS ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -243,6 +320,7 @@ get_cipher_pref() { # If the connection succeeded with the current cipher, benchmark and store if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") + ciphercertificates=("${ciphercertificates[@]}" "$certificates") pciph=$(echo $result|awk '{print $1}') get_cipher_pref "!$pciph:$ciphersuite" return 0 @@ -349,6 +427,9 @@ display_results_in_json() { echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + if [[ -n $CAPATH ]]; then + echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," + fi echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," pfs=$(echo $cipher|awk '{print $8}') @@ -393,7 +474,7 @@ test_serverside_ordering() { local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e "$CACERTS" ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -453,6 +534,10 @@ do CAPATH="$2" shift 2 ;; + --saveca) + SAVECA="True" + shift 1 + ;; --) # End of all options shift break @@ -505,6 +590,7 @@ debug "sclientargs: $SCLIENTARGS" cipherspref=(); +ciphercertificates=() results=() # Call to the recursive loop that retrieves the cipher preferences From 3e37517c966396103c25fe584bff589364e92cef Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 15:18:48 +0200 Subject: [PATCH 3/9] add ability to also save leaf certificates and untrusted ones --- cipherscan | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index 02831e7..ae2ca2b 100755 --- a/cipherscan +++ b/cipherscan @@ -47,6 +47,7 @@ TIMEOUT=30 # place where to put the found intermediate CA certificates and where # trust anchors are stored CAPATH="" +SAVECRT="" # because running external commands like sleep incurs a fork penalty, we # first check if it is necessary @@ -57,7 +58,7 @@ ratelimit() { } usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -78,6 +79,7 @@ Use one of the options below: -h | --help Shows this help text. -j | --json Output results in JSON format. -o | --openssl path/to/your/openssl binary you want to use. +--savecrt path where to save untrusted and leaf certificates -v | --verbose Increase verbosity. The rest of the arguments will be interpreted as openssl s_client argument. @@ -209,6 +211,7 @@ test_cipher_on_target() { # check if the certificate is actually trusted (server may present # unrelated certificates that are not trusted (including self # signed ones) + local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ grep 'OK$' >/dev/null; then @@ -220,6 +223,12 @@ test_cipher_on_target() { echo "$cert" > "$CAPATH/${sha256sum}.pem" c_hash "$CAPATH" "${sha256sum}.pem" fi + saved="True" + fi + fi + if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then + if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then + echo "$cert" > "$SAVECRT/${sha256sum}.pem" fi fi # save the sha sum for reporting @@ -538,6 +547,10 @@ do SAVECA="True" shift 1 ;; + --savecrt) + SAVECRT="$2" + shift 2 + ;; --) # End of all options shift break From d9b718be129b4c37f2df2151d73bc78e542e66b1 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 11 Jul 2014 17:29:57 +0200 Subject: [PATCH 4/9] clean up the extracted certificate the certificate extracted in the above way will contain some junk from openssl s_client output we don't want like verification status we can remove it ro reduce disk usage for saved certificates --- cipherscan | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cipherscan b/cipherscan index ae2ca2b..1a2236b 100755 --- a/cipherscan +++ b/cipherscan @@ -188,6 +188,8 @@ test_cipher_on_target() { /-----END CERTIFICATE-----/ {split_after=1} {if (n == i) print } ' <<<"$tmp") + # clean up the cert from junk before BEGIN CERTIFICATE + cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) # compute sha256 fingerprint of the certificate local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ From 1eae0cc71ba459525ccf9564c5d092daf1e2b9a9 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 14:15:59 +0200 Subject: [PATCH 5/9] use CApath for certificates and store certificates (v2) CApath is about 20% faster than CAfile so use it, also save the received certificates from the servers for later analysis (proper hostname checking, looking for certificates sharing private key, etc.) Use the mechanism from cipherscan to find location of ca cert bundle --- top1m/testtop1m.sh | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index e3e6920..8a39055 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -9,6 +9,42 @@ if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then exit 1 fi [ ! -e "results" ] && mkdir results +[ ! -e "certs" ] && mkdir certs +if [ -z "$CACERTS" ]; then + for f in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do + if [ -e "$f" ]; then + CACERTS="$f" + break + fi + done +fi +if [ ! -e "$CACERTS" ]; then + echo "file with CA certificates does not exist, please export CACERTS variable with location" + exit 1 +fi +if [ ! -e "ca_files" ]; then + mkdir ca_files + pushd ca_files >/dev/null + awk ' + split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {print > "cert" n ".pem"}' < "$CACERTS" + for i in *; do + h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null) + for num in `seq 0 100`; do + if [[ $h.$num -ef $i ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $h.$num ]]; then + # file doesn't exist, create a link + ln -s "$i" "$h.$num" + break + fi + done + done + popd >/dev/null +fi function wait_for_jobs() { local no_jobs @@ -32,7 +68,7 @@ function scan_host() { if [ $? -gt 0 ]; then return fi - ../cipherscan --delay 2 -json -servername $1 $2:443 > results/$1@$2 + ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json -servername $1 $2:443 > results/$1@$2 } function scan_host_no_sni() { @@ -44,7 +80,7 @@ function scan_host_no_sni() { if [ $? -gt 0 ]; then return fi - ../cipherscan --delay 2 -json $1:443 > results/$1 + ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json $1:443 > results/$1 } function scan_hostname() { From 0f576c1fbcd7d16f881bc251c329c9ec7899c8c7 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:17:52 +0200 Subject: [PATCH 6/9] don't calculate sha sums for the certificates over and over we can use cksum to calculate simple checksum much faster than with using openssl, so we can compute sums only once --- cipherscan | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cipherscan b/cipherscan index 1a2236b..ddf4667 100755 --- a/cipherscan +++ b/cipherscan @@ -48,6 +48,10 @@ TIMEOUT=30 # trust anchors are stored CAPATH="" SAVECRT="" +unset known_certs +declare -A known_certs +unset cert_checksums +declare -A cert_checksums # because running external commands like sleep incurs a fork penalty, we # first check if it is necessary @@ -184,15 +188,25 @@ test_cipher_on_target() { for ((i=0; i<$certificate_count; i=i+1 )); do # extract i'th certificate - local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} - /-----END CERTIFICATE-----/ {split_after=1} - {if (n == i) print } - ' <<<"$tmp") - # clean up the cert from junk before BEGIN CERTIFICATE - cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) + local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } + /-----BEGIN CERTIFICATE-----/ { output=1 } + output==1 { if (n==i) print } + /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + # put the output to an array instead awk '{print $1}' + local cksum=($(cksum <<<"$cert")) + # compare the values not just checksums so that eventual collision + # doesn't mess up results + if [[ ${known_certs[$cksum]} == $cert ]]; then + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${cert_checksums[$cksum]}\"" + continue + fi # compute sha256 fingerprint of the certificate - local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + <<<"$cert" 2>/dev/null |\ ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') # check if it is a CA certificate @@ -238,6 +252,8 @@ test_cipher_on_target() { current_certificates+="," fi current_certificates+="\"${sha256sum}\"" + known_certs[$cksum]="$cert" + cert_checksums[$cksum]="$sha256sum" done debug "current_certificates: $current_certificates" From 4c22d50f0c523d35d013cd2a2633139f93e65648 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:54:33 +0200 Subject: [PATCH 7/9] few less forks in the script again, we can use arrays and a bit advanced awk syntax to reduce the number of forks necessary to run the script --- cipherscan | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cipherscan b/cipherscan index ddf4667..9bae63d 100755 --- a/cipherscan +++ b/cipherscan @@ -192,7 +192,7 @@ test_cipher_on_target() { /-----BEGIN CERTIFICATE-----/ { output=1 } output==1 { if (n==i) print } /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") - # put the output to an array instead awk '{print $1}' + # put the output to an array instead running awk '{print $1}' local cksum=($(cksum <<<"$cert")) # compare the values not just checksums so that eventual collision # doesn't mess up results @@ -348,7 +348,7 @@ get_cipher_pref() { if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") ciphercertificates=("${ciphercertificates[@]}" "$certificates") - pciph=$(echo $result|awk '{print $1}') + pciph=($(echo $result)) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -365,7 +365,7 @@ display_results_in_terminal() { local different=False echo "Target: $TARGET"; echo for cipher in "${cipherspref[@]}"; do - pciph=$(echo $cipher|awk '{print $1}') + pciph=($(echo $cipher)) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" @@ -419,7 +419,7 @@ display_results_in_terminal() { echo $result|grep -v '(NONE)' else # prints priority, ciphersuite, protocols and pfs_keysize - echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}' + awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result" fi done|column -t echo From 9f06829486e9fd5c58d01591c6a818c1038993f2 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 4 Aug 2014 17:22:53 +0200 Subject: [PATCH 8/9] make handling of self signed certs more robust openssl sometimes will print the filename, then the error, and finish with OK, matching the colon and space prevents from considering such certs to be valid --- cipherscan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index 9bae63d..b62ebb1 100755 --- a/cipherscan +++ b/cipherscan @@ -230,7 +230,7 @@ test_cipher_on_target() { local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ - grep 'OK$' >/dev/null; then + grep ': OK$' >/dev/null; then # if the certificate is an intermediate CA it may be useful # for connecting to servers that are misconfigured so save it From c4a8495a54c1fe03bde6e3484ff7f401068e9dda Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 18 Oct 2014 17:20:20 +0200 Subject: [PATCH 9/9] limit number of forks needed to speed up execution bash has a built in regular expression processor, we can match lines using =~ moreover, stuff that will match while being inside parentheses is later available in the BASH_REMATCH array the IFS (Internal Field Separator) by default includes space, tab and new line, as such we can use it to split longer lines to separate words, just as awk '{print $1}' can, just need to put the value to an array for that we also don't have to use $(echo $var) when assigning variables, $var is enough bash has also built in substitution engine, so we can do ${var/,/ & } to switch all commas to ampersands when using the variable --- cipherscan | 217 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 146 insertions(+), 71 deletions(-) diff --git a/cipherscan b/cipherscan index b62ebb1..6efb49c 100755 --- a/cipherscan +++ b/cipherscan @@ -62,7 +62,9 @@ ratelimit() { } usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] +[--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] +[-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -108,7 +110,7 @@ debug(){ c_hash() { local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) - for num in $(seq 0 100); do + for ((num=0; num<=100; num++)) ; do if [[ $1/${h}.${num} -ef $2 ]]; then # file already linked, ignore break @@ -123,6 +125,106 @@ c_hash() { done } +parse_openssl_output() { + # clear variables in case matching doesn't hit them + current_ocspstaple="False" + current_cipher="" + current_pfs="" + current_protocol="" + current_tickethint="None" + current_pubkey=0 + current_trusted="False" + current_sigalg="None" + + certs_found=0 + current_raw_certificates=() + + while read line; do + # check if there isn't OCSP response data (response and responder cert) + if [[ $line =~ ^====================================== ]]; then + while read data; do + # check if there is a OCSP response in output + if [[ $data =~ OCSP\ Response\ Data ]]; then + current_ocspstaple="True" + continue + fi + + # skip all data from a OCSP response + if [[ $data =~ ^====================================== ]]; then + break + fi + done + continue + fi + + # extract selected cipher + if [[ $line =~ New,\ ]]; then + local match=($line) + current_cipher="${match[4]}" + continue + fi + + # extract data about selected temporary key + if [[ $line =~ Server\ Temp\ Key ]]; then + local match=($line) + current_pfs="${match[3]}${match[4]}${match[5]}${match[6]}" + continue + fi + + # extract used protocol + if [[ $line =~ ^Protocol\ + ]]; then + local match=($line) + current_protocol="${match[2]}" + continue + fi + + # extract session ticket hint + if [[ $line =~ ticket\ lifetime\ hint ]]; then + local match=($line) + current_tickethint="${match[5]}" + continue + fi + + # extract size of server public key + if [[ $line =~ Server\ public\ key\ is\ ]]; then + local match=($line) + current_pubkey="${match[4]}" + continue + fi + + # check if connection used trused certificate + if [[ $line =~ Verify\ return\ code:\ 0 ]]; then + current_trusted="True" + continue + fi + + # extract certificates + if [[ $line =~ -----BEGIN\ CERTIFICATE----- ]]; then + current_raw_certificates[$certs_found]="$line"$'\n' + while read data; do + current_raw_certificates[$certs_found]+="$data"$'\n' + if [[ $data =~ -----END\ CERTIFICATE----- ]]; then + break + fi + done + certs_found=$((certs_found+1)) + continue + fi + done + + # if we found any certs in output, process the first one and extract + # the signature algorithm on it (it's the server's certificate) + if [[ $certs_found -gt 0 ]]; then + local ossl_out=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"${current_raw_certificates[0]}") + while read data; do + if [[ $data =~ Signature\ Algorithm ]]; then + local match=($data) + current_sigalg="${match[2]}" + fi + done <<<"$ossl_out" + fi +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -138,60 +240,31 @@ test_cipher_on_target() { # in SSLv3 mode OpenSSL just ignores the setting so it's ok # -status exception is ignored in SSLv2, go figure if [ "$tls_version" == "-ssl2" ]; then - cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand) + if [[ "$sslcommand" =~ (.*)(-servername\ [^ ]*)(.*) ]]; then + cmnd="${BASH_REMATCH[1]} ${BASH_REMATCH[3]}" + else + cmnd="$sslcommand" + fi else cmnd=$sslcommand fi ratelimit debug echo \"Q\" \| $cmnd $tls_version local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null) - if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then - current_ocspstaple="True" - else - current_ocspstaple="False" - fi - # filter out the OCSP server certificate - tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") - # session metadata - current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") - current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") - current_protocol=$(awk '/^ +Protocol +:/ {print $3; exit}' <<<"$tmp") - current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") - if [ -z $current_tickethint ]; then - current_tickethint=None - fi - - # certificate metadata - current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") - if [ -z $current_pubkey ]; then - current_pubkey=0 - fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ - awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" - grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null - if [ $? -eq 0 ]; then - current_trusted="True" - else - current_trusted="False" - fi - if [ -z $current_sigalg ]; then - current_sigalg=None - fi + parse_openssl_output <<<"$tmp" + verbose "selected cipher is '$current_cipher'" + verbose "using protocol '$current_protocol'" # collect certificate data current_certificates="" - local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ - <<<"$tmp") + local certificate_count=$certs_found debug "server presented $certificate_count certificates" local i for ((i=0; i<$certificate_count; i=i+1 )); do # extract i'th certificate - local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } - /-----BEGIN CERTIFICATE-----/ { output=1 } - output==1 { if (n==i) print } - /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + local cert="${current_raw_certificates[$i]}" # put the output to an array instead running awk '{print $1}' local cksum=($(cksum <<<"$cert")) # compare the values not just checksums so that eventual collision @@ -205,9 +278,9 @@ test_cipher_on_target() { fi # compute sha256 fingerprint of the certificate - local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + local sha256sum=($(${OPENSSLBIN} x509 -outform DER\ <<<"$cert" 2>/dev/null |\ - ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null)) # check if it is a CA certificate local isCA="False" @@ -229,7 +302,7 @@ test_cipher_on_target() { # signed ones) local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ - -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + -untrusted <(printf "%s" "${current_raw_certificates[@]}") <(echo "$cert") 2>/dev/null | \ grep ': OK$' >/dev/null; then # if the certificate is an intermediate CA it may be useful @@ -348,7 +421,7 @@ get_cipher_pref() { if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") ciphercertificates=("${ciphercertificates[@]}" "$certificates") - pciph=($(echo $result)) + pciph=($result) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -365,30 +438,32 @@ display_results_in_terminal() { local different=False echo "Target: $TARGET"; echo for cipher in "${cipherspref[@]}"; do - pciph=($(echo $cipher)) + # get first in array + pciph=($cipher) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" else r="$ctr $cipher" fi + local cipher_data=($cipher) if [ $ctr -eq 1 ]; then - pubkey=$(awk '{print $3}' <<<$cipher) - sigalg=$(awk '{print $4}' <<<$cipher) - trusted=$(awk '{print $5}' <<<$cipher) - tickethint=$(awk '{print $6}' <<<$cipher) - ocspstaple=$(awk '{print $7}' <<<$cipher) + pubkey="${cipher_data[2]}" + sigalg="${cipher_data[3]}" + trusted="${cipher_data[4]}" + tickethint="${cipher_data[5]}" + ocspstaple="${cipher_data[6]}" else - if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then + if [ "$pubkey" != "${cipher_data[2]}" ]; then different=True fi - if [ "$sigalg" != "$(awk '{print $4}' <<<$cipher)" ]; then + if [ "$sigalg" != "${cipher_data[3]}" ]; then different=True fi - if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then + if [ "$trusted" != "${cipher_data[4]}" ]; then different=True fi - if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then + if [ "$tickethint" != "${cipher_data[5]}" ]; then different=True fi fi @@ -448,18 +523,19 @@ display_results_in_json() { ctr=0 echo -n "{\"target\":\"$TARGET\",\"utctimestamp\":\"$(date -u '+%FT%T.0Z')\",\"serverside\":\"${serverside}\",\"ciphersuite\": [" for cipher in "${cipherspref[@]}"; do + local cipher_arr=($cipher) [ $ctr -gt 0 ] && echo -n ',' - echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\"," - echo -n "\"protocols\":[\"$(echo $cipher|awk '{print $2}'|sed 's/,/","/g')\"]," - echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," - echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," - echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + echo -n "{\"cipher\":\"${cipher_arr[0]}\"," + echo -n "\"protocols\":[\"${cipher_arr[1]//,/\",\"}\"]," + echo -n "\"pubkey\":[\"${cipher_arr[2]//,/\",\"}\"]," + echo -n "\"sigalg\":[\"${cipher_arr[3]//,/\",\"}\"]," + echo -n "\"trusted\":\"${cipher_arr[4]//,/\",\"}\"," if [[ -n $CAPATH ]]; then echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," fi - echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," - echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," - pfs=$(echo $cipher|awk '{print $8}') + echo -n "\"ticket_hint\":\"${cipher_arr[5]}\"," + echo -n "\"ocsp_stapling\":\"${cipher_arr[6]}\"," + pfs="${cipher_arr[7]}" [ "$pfs" == "" ] && pfs="None" echo -n "\"pfs\":\"$pfs\"}" ctr=$((ctr+1)) @@ -477,25 +553,25 @@ test_serverside_ordering() { # server supports just two ciphers, so rotate them, that should be enough elif [[ ${#cipherspref[@]} -eq 2 ]]; then - local cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + local cipher=(${cipherspref[1]}) prefered="$cipher" ciphersuite=$cipher - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" # server supports 3 or more ciphers, rotate all three. This is necessary because google does # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or # ECDHE-RSA-CHACHA20-POLY1305 else - local cipher=$(awk '{print $1}' <<< ${cipherspref[2]}) + local cipher=(${cipherspref[2]}) prefered="$cipher" ciphersuite="$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + cipher=(${cipherspref[1]}) ciphersuite+=":$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" fi @@ -511,7 +587,7 @@ test_serverside_ordering() { if [ $? -ne 0 ]; then serverside="True" else - local selected=$(awk '{print $1}' <<< $result) + local selected=($result) if [[ $selected == $prefered ]]; then serverside="False" else @@ -608,8 +684,7 @@ TARGET=$HOST:$PORT debug "target: $TARGET" # test our openssl is usable -tmp="$($OPENSSLBIN -h 2>&1 1>/dev/null)" -if [ $? -gt 0 ]; then +if [ ! -x $OPENSSLBIN ]; then OPENSSLBIN=$(which openssl) if [ "$OUTPUTFORMAT" == "terminal" ]; then echo "custom openssl not executable, falling back to system one from $OPENSSLBIN"