From 81a8ac0253eb23732f388f498ae1cdfcb0774223 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 13:42:48 +0200 Subject: [PATCH 01/10] no need to grep the input when we're using awk (v2) awk has an inbuilt version of grep, also truncate processing as soon as we find what we're looking for This version uses slightly different syntax that is compatible with old awk --- cipherscan | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cipherscan b/cipherscan index 3990ea6..c2c1a5b 100755 --- a/cipherscan +++ b/cipherscan @@ -129,20 +129,21 @@ test_cipher_on_target() { tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") # session metadata - current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') - current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') - current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') - current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }') + current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") + current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") + current_protocol=$(awk '/^ +Protocol +:/ {print $3; exit}' <<<"$tmp") + current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") if [ -z $current_tickethint ]; then current_tickethint=None fi # certificate metadata - current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') + current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") if [ -z $current_pubkey ]; then current_pubkey=0 fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" + current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ + awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null if [ $? -eq 0 ]; then current_trusted="True" From 605a1b85fedacfac3bde7242f8d8b64e3b581a2b Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 15:18:11 +0200 Subject: [PATCH 02/10] add caching of intermediate CA certificates --- cipherscan | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/cipherscan b/cipherscan index c2c1a5b..02831e7 100755 --- a/cipherscan +++ b/cipherscan @@ -57,7 +57,7 @@ ratelimit() { } usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -71,7 +71,8 @@ Use one of the options below: -a | --allciphers Test all known ciphers individually at the end. -b | --benchmark Activate benchmark mode. ---capath use CAs from directory +--capath use CAs from directory (must be in OpenSSL CAdir format) +--saveca save intermediate certificates in CA directory -d | --delay Pause for n seconds between connections -D | --debug Output ALL the information. -h | --help Shows this help text. @@ -99,6 +100,23 @@ debug(){ fi } +c_hash() { + local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) + for num in $(seq 0 100); do + if [[ $1/${h}.${num} -ef $2 ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $1/${h}.${num} ]]; then + # file doesn't exist, create a link + pushd "$1" > /dev/null + ln -s "$2" "${h}.${num}" + popd > /dev/null + break + fi + done +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -107,6 +125,7 @@ test_cipher_on_target() { protocols="" pfs="" previous_cipher="" + certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do # sslv2 client hello doesn't support SNI extension @@ -154,6 +173,63 @@ test_cipher_on_target() { current_sigalg=None fi + # collect certificate data + current_certificates="" + local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ + <<<"$tmp") + debug "server presented $certificate_count certificates" + local i + for ((i=0; i<$certificate_count; i=i+1 )); do + + # extract i'th certificate + local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {if (n == i) print } + ' <<<"$tmp") + + # compute sha256 fingerprint of the certificate + local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + + # check if it is a CA certificate + local isCA="False" + if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\ + grep 'CA:TRUE' >/dev/null; then + isCA="True" + fi + + # build trust source for certificate verification + local trust_source=() + if [[ -n $CAPATH ]]; then + trust_source=("-CApath" "$CAPATH") + elif [[ -e $CACERTS ]]; then + trust_source=("-CAfile" "$CACERTS") + fi + + # check if the certificate is actually trusted (server may present + # unrelated certificates that are not trusted (including self + # signed ones) + if ${OPENSSLBIN} verify "${trust_source[@]}" \ + -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + grep 'OK$' >/dev/null; then + + # if the certificate is an intermediate CA it may be useful + # for connecting to servers that are misconfigured so save it + if [[ -n $CAPATH ]] && [[ $SAVECA == "True" ]] && [[ $isCA == "True" ]]; then + if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then + echo "$cert" > "$CAPATH/${sha256sum}.pem" + c_hash "$CAPATH" "${sha256sum}.pem" + fi + fi + fi + # save the sha sum for reporting + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${sha256sum}\"" + done + debug "current_certificates: $current_certificates" + # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then # connection failed, try again with next TLS version @@ -180,6 +256,7 @@ test_cipher_on_target() { trusted=$current_trusted tickethint=$current_tickethint ocspstaple=$current_ocspstaple + certificates="$current_certificates" # grab the cipher and PFS key size done # if cipher is empty, that means none of the TLS version worked with @@ -231,7 +308,7 @@ get_cipher_pref() { local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e $CACERTS ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -243,6 +320,7 @@ get_cipher_pref() { # If the connection succeeded with the current cipher, benchmark and store if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") + ciphercertificates=("${ciphercertificates[@]}" "$certificates") pciph=$(echo $result|awk '{print $1}') get_cipher_pref "!$pciph:$ciphersuite" return 0 @@ -349,6 +427,9 @@ display_results_in_json() { echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + if [[ -n $CAPATH ]]; then + echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," + fi echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," pfs=$(echo $cipher|awk '{print $8}') @@ -393,7 +474,7 @@ test_serverside_ordering() { local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e "$CACERTS" ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -453,6 +534,10 @@ do CAPATH="$2" shift 2 ;; + --saveca) + SAVECA="True" + shift 1 + ;; --) # End of all options shift break @@ -505,6 +590,7 @@ debug "sclientargs: $SCLIENTARGS" cipherspref=(); +ciphercertificates=() results=() # Call to the recursive loop that retrieves the cipher preferences From 7087bdb5a03299520512d2e3180b9ece906ca25d Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 15:18:48 +0200 Subject: [PATCH 03/10] add ability to also save leaf certificates and untrusted ones --- cipherscan | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index 02831e7..ae2ca2b 100755 --- a/cipherscan +++ b/cipherscan @@ -47,6 +47,7 @@ TIMEOUT=30 # place where to put the found intermediate CA certificates and where # trust anchors are stored CAPATH="" +SAVECRT="" # because running external commands like sleep incurs a fork penalty, we # first check if it is necessary @@ -57,7 +58,7 @@ ratelimit() { } usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -78,6 +79,7 @@ Use one of the options below: -h | --help Shows this help text. -j | --json Output results in JSON format. -o | --openssl path/to/your/openssl binary you want to use. +--savecrt path where to save untrusted and leaf certificates -v | --verbose Increase verbosity. The rest of the arguments will be interpreted as openssl s_client argument. @@ -209,6 +211,7 @@ test_cipher_on_target() { # check if the certificate is actually trusted (server may present # unrelated certificates that are not trusted (including self # signed ones) + local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ grep 'OK$' >/dev/null; then @@ -220,6 +223,12 @@ test_cipher_on_target() { echo "$cert" > "$CAPATH/${sha256sum}.pem" c_hash "$CAPATH" "${sha256sum}.pem" fi + saved="True" + fi + fi + if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then + if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then + echo "$cert" > "$SAVECRT/${sha256sum}.pem" fi fi # save the sha sum for reporting @@ -538,6 +547,10 @@ do SAVECA="True" shift 1 ;; + --savecrt) + SAVECRT="$2" + shift 2 + ;; --) # End of all options shift break From 62808a33c8ae4779bbca7f51ad83d8cbab074f45 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 11 Jul 2014 17:29:57 +0200 Subject: [PATCH 04/10] clean up the extracted certificate the certificate extracted in the above way will contain some junk from openssl s_client output we don't want like verification status we can remove it ro reduce disk usage for saved certificates --- cipherscan | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cipherscan b/cipherscan index ae2ca2b..1a2236b 100755 --- a/cipherscan +++ b/cipherscan @@ -188,6 +188,8 @@ test_cipher_on_target() { /-----END CERTIFICATE-----/ {split_after=1} {if (n == i) print } ' <<<"$tmp") + # clean up the cert from junk before BEGIN CERTIFICATE + cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) # compute sha256 fingerprint of the certificate local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ From 68da6f00b6b502550aacaa6815d5fe9427460a33 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 14:15:59 +0200 Subject: [PATCH 05/10] use CApath for certificates and store certificates (v2) CApath is about 20% faster than CAfile so use it, also save the received certificates from the servers for later analysis (proper hostname checking, looking for certificates sharing private key, etc.) Use the mechanism from cipherscan to find location of ca cert bundle --- top1m/testtop1m.sh | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index e3e6920..8a39055 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -9,6 +9,42 @@ if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then exit 1 fi [ ! -e "results" ] && mkdir results +[ ! -e "certs" ] && mkdir certs +if [ -z "$CACERTS" ]; then + for f in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do + if [ -e "$f" ]; then + CACERTS="$f" + break + fi + done +fi +if [ ! -e "$CACERTS" ]; then + echo "file with CA certificates does not exist, please export CACERTS variable with location" + exit 1 +fi +if [ ! -e "ca_files" ]; then + mkdir ca_files + pushd ca_files >/dev/null + awk ' + split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {print > "cert" n ".pem"}' < "$CACERTS" + for i in *; do + h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null) + for num in `seq 0 100`; do + if [[ $h.$num -ef $i ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $h.$num ]]; then + # file doesn't exist, create a link + ln -s "$i" "$h.$num" + break + fi + done + done + popd >/dev/null +fi function wait_for_jobs() { local no_jobs @@ -32,7 +68,7 @@ function scan_host() { if [ $? -gt 0 ]; then return fi - ../cipherscan --delay 2 -json -servername $1 $2:443 > results/$1@$2 + ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json -servername $1 $2:443 > results/$1@$2 } function scan_host_no_sni() { @@ -44,7 +80,7 @@ function scan_host_no_sni() { if [ $? -gt 0 ]; then return fi - ../cipherscan --delay 2 -json $1:443 > results/$1 + ../cipherscan --capath ca_files --saveca --savecrt certs --delay 2 -json $1:443 > results/$1 } function scan_hostname() { From 413a43ecbfb5caf21777603d6e5e4c79aa36e32f Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:17:52 +0200 Subject: [PATCH 06/10] don't calculate sha sums for the certificates over and over we can use cksum to calculate simple checksum much faster than with using openssl, so we can compute sums only once --- cipherscan | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cipherscan b/cipherscan index 1a2236b..ddf4667 100755 --- a/cipherscan +++ b/cipherscan @@ -48,6 +48,10 @@ TIMEOUT=30 # trust anchors are stored CAPATH="" SAVECRT="" +unset known_certs +declare -A known_certs +unset cert_checksums +declare -A cert_checksums # because running external commands like sleep incurs a fork penalty, we # first check if it is necessary @@ -184,15 +188,25 @@ test_cipher_on_target() { for ((i=0; i<$certificate_count; i=i+1 )); do # extract i'th certificate - local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} - /-----END CERTIFICATE-----/ {split_after=1} - {if (n == i) print } - ' <<<"$tmp") - # clean up the cert from junk before BEGIN CERTIFICATE - cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) + local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } + /-----BEGIN CERTIFICATE-----/ { output=1 } + output==1 { if (n==i) print } + /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + # put the output to an array instead awk '{print $1}' + local cksum=($(cksum <<<"$cert")) + # compare the values not just checksums so that eventual collision + # doesn't mess up results + if [[ ${known_certs[$cksum]} == $cert ]]; then + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${cert_checksums[$cksum]}\"" + continue + fi # compute sha256 fingerprint of the certificate - local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + <<<"$cert" 2>/dev/null |\ ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') # check if it is a CA certificate @@ -238,6 +252,8 @@ test_cipher_on_target() { current_certificates+="," fi current_certificates+="\"${sha256sum}\"" + known_certs[$cksum]="$cert" + cert_checksums[$cksum]="$sha256sum" done debug "current_certificates: $current_certificates" From 911b3e2515868b616db5a9790d0767a86e9ca940 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:54:33 +0200 Subject: [PATCH 07/10] few less forks in the script again, we can use arrays and a bit advanced awk syntax to reduce the number of forks necessary to run the script --- cipherscan | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cipherscan b/cipherscan index ddf4667..9bae63d 100755 --- a/cipherscan +++ b/cipherscan @@ -192,7 +192,7 @@ test_cipher_on_target() { /-----BEGIN CERTIFICATE-----/ { output=1 } output==1 { if (n==i) print } /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") - # put the output to an array instead awk '{print $1}' + # put the output to an array instead running awk '{print $1}' local cksum=($(cksum <<<"$cert")) # compare the values not just checksums so that eventual collision # doesn't mess up results @@ -348,7 +348,7 @@ get_cipher_pref() { if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") ciphercertificates=("${ciphercertificates[@]}" "$certificates") - pciph=$(echo $result|awk '{print $1}') + pciph=($(echo $result)) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -365,7 +365,7 @@ display_results_in_terminal() { local different=False echo "Target: $TARGET"; echo for cipher in "${cipherspref[@]}"; do - pciph=$(echo $cipher|awk '{print $1}') + pciph=($(echo $cipher)) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" @@ -419,7 +419,7 @@ display_results_in_terminal() { echo $result|grep -v '(NONE)' else # prints priority, ciphersuite, protocols and pfs_keysize - echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}' + awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result" fi done|column -t echo From 94a1c6fc0d8af62aef1f7604beda5b06851eaefa Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 4 Aug 2014 17:22:53 +0200 Subject: [PATCH 08/10] make handling of self signed certs more robust openssl sometimes will print the filename, then the error, and finish with OK, matching the colon and space prevents from considering such certs to be valid --- cipherscan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index 9bae63d..b62ebb1 100755 --- a/cipherscan +++ b/cipherscan @@ -230,7 +230,7 @@ test_cipher_on_target() { local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ - grep 'OK$' >/dev/null; then + grep ': OK$' >/dev/null; then # if the certificate is an intermediate CA it may be useful # for connecting to servers that are misconfigured so save it From 26204115bf5382ac6304fea3f1785caf21abf317 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 01:30:17 +0200 Subject: [PATCH 09/10] don't retry protocols we know don't work When connection is unsuccessful with a given protocol, don't try it again since we probably exhausted the ciphers supported by the protocol makes scanning about 10% faster --- cipherscan | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cipherscan b/cipherscan index b62ebb1..cceaa3f 100755 --- a/cipherscan +++ b/cipherscan @@ -52,6 +52,8 @@ unset known_certs declare -A known_certs unset cert_checksums declare -A cert_checksums +unset ok_protocols +declare -A ok_protocols # because running external commands like sleep incurs a fork penalty, we # first check if it is necessary @@ -134,6 +136,9 @@ test_cipher_on_target() { certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do + if [[ ${ok_protocols[$tls_version]} -eq 1 ]]; then + continue + fi # sslv2 client hello doesn't support SNI extension # in SSLv3 mode OpenSSL just ignores the setting so it's ok # -status exception is ignored in SSLv2, go figure @@ -259,6 +264,7 @@ test_cipher_on_target() { # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then + ok_protocols["$tls_version"]=1 # connection failed, try again with next TLS version continue else @@ -626,6 +632,8 @@ results=() # Call to the recursive loop that retrieves the cipher preferences get_cipher_pref $CIPHERSUITE +unset ok_protocols +declare -A ok_protocols test_serverside_ordering From f1faa28a8c164888d9477f7e69225c01c9ede1a7 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 18 Oct 2014 17:20:20 +0200 Subject: [PATCH 10/10] limit number of forks needed to speed up execution bash has a built in regular expression processor, we can match lines using =~ moreover, stuff that will match while being inside parentheses is later available in the BASH_REMATCH array the IFS (Internal Field Separator) by default includes space, tab and new line, as such we can use it to split longer lines to separate words, just as awk '{print $1}' can, just need to put the value to an array for that we also don't have to use $(echo $var) when assigning variables, $var is enough bash has also built in substitution engine, so we can do ${var/,/ & } to switch all commas to ampersands when using the variable --- cipherscan | 217 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 146 insertions(+), 71 deletions(-) diff --git a/cipherscan b/cipherscan index cceaa3f..6b62d2a 100755 --- a/cipherscan +++ b/cipherscan @@ -64,7 +64,9 @@ ratelimit() { } usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] +[--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] +[-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -110,7 +112,7 @@ debug(){ c_hash() { local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) - for num in $(seq 0 100); do + for ((num=0; num<=100; num++)) ; do if [[ $1/${h}.${num} -ef $2 ]]; then # file already linked, ignore break @@ -125,6 +127,106 @@ c_hash() { done } +parse_openssl_output() { + # clear variables in case matching doesn't hit them + current_ocspstaple="False" + current_cipher="" + current_pfs="" + current_protocol="" + current_tickethint="None" + current_pubkey=0 + current_trusted="False" + current_sigalg="None" + + certs_found=0 + current_raw_certificates=() + + while read line; do + # check if there isn't OCSP response data (response and responder cert) + if [[ $line =~ ^====================================== ]]; then + while read data; do + # check if there is a OCSP response in output + if [[ $data =~ OCSP\ Response\ Data ]]; then + current_ocspstaple="True" + continue + fi + + # skip all data from a OCSP response + if [[ $data =~ ^====================================== ]]; then + break + fi + done + continue + fi + + # extract selected cipher + if [[ $line =~ New,\ ]]; then + local match=($line) + current_cipher="${match[4]}" + continue + fi + + # extract data about selected temporary key + if [[ $line =~ Server\ Temp\ Key ]]; then + local match=($line) + current_pfs="${match[3]}${match[4]}${match[5]}${match[6]}" + continue + fi + + # extract used protocol + if [[ $line =~ ^Protocol\ + ]]; then + local match=($line) + current_protocol="${match[2]}" + continue + fi + + # extract session ticket hint + if [[ $line =~ ticket\ lifetime\ hint ]]; then + local match=($line) + current_tickethint="${match[5]}" + continue + fi + + # extract size of server public key + if [[ $line =~ Server\ public\ key\ is\ ]]; then + local match=($line) + current_pubkey="${match[4]}" + continue + fi + + # check if connection used trused certificate + if [[ $line =~ Verify\ return\ code:\ 0 ]]; then + current_trusted="True" + continue + fi + + # extract certificates + if [[ $line =~ -----BEGIN\ CERTIFICATE----- ]]; then + current_raw_certificates[$certs_found]="$line"$'\n' + while read data; do + current_raw_certificates[$certs_found]+="$data"$'\n' + if [[ $data =~ -----END\ CERTIFICATE----- ]]; then + break + fi + done + certs_found=$((certs_found+1)) + continue + fi + done + + # if we found any certs in output, process the first one and extract + # the signature algorithm on it (it's the server's certificate) + if [[ $certs_found -gt 0 ]]; then + local ossl_out=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"${current_raw_certificates[0]}") + while read data; do + if [[ $data =~ Signature\ Algorithm ]]; then + local match=($data) + current_sigalg="${match[2]}" + fi + done <<<"$ossl_out" + fi +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -143,60 +245,31 @@ test_cipher_on_target() { # in SSLv3 mode OpenSSL just ignores the setting so it's ok # -status exception is ignored in SSLv2, go figure if [ "$tls_version" == "-ssl2" ]; then - cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand) + if [[ "$sslcommand" =~ (.*)(-servername\ [^ ]*)(.*) ]]; then + cmnd="${BASH_REMATCH[1]} ${BASH_REMATCH[3]}" + else + cmnd="$sslcommand" + fi else cmnd=$sslcommand fi ratelimit debug echo \"Q\" \| $cmnd $tls_version local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null) - if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then - current_ocspstaple="True" - else - current_ocspstaple="False" - fi - # filter out the OCSP server certificate - tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") - # session metadata - current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") - current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") - current_protocol=$(awk '/^ +Protocol +:/ {print $3; exit}' <<<"$tmp") - current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") - if [ -z $current_tickethint ]; then - current_tickethint=None - fi - - # certificate metadata - current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") - if [ -z $current_pubkey ]; then - current_pubkey=0 - fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ - awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" - grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null - if [ $? -eq 0 ]; then - current_trusted="True" - else - current_trusted="False" - fi - if [ -z $current_sigalg ]; then - current_sigalg=None - fi + parse_openssl_output <<<"$tmp" + verbose "selected cipher is '$current_cipher'" + verbose "using protocol '$current_protocol'" # collect certificate data current_certificates="" - local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ - <<<"$tmp") + local certificate_count=$certs_found debug "server presented $certificate_count certificates" local i for ((i=0; i<$certificate_count; i=i+1 )); do # extract i'th certificate - local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } - /-----BEGIN CERTIFICATE-----/ { output=1 } - output==1 { if (n==i) print } - /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + local cert="${current_raw_certificates[$i]}" # put the output to an array instead running awk '{print $1}' local cksum=($(cksum <<<"$cert")) # compare the values not just checksums so that eventual collision @@ -210,9 +283,9 @@ test_cipher_on_target() { fi # compute sha256 fingerprint of the certificate - local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + local sha256sum=($(${OPENSSLBIN} x509 -outform DER\ <<<"$cert" 2>/dev/null |\ - ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null)) # check if it is a CA certificate local isCA="False" @@ -234,7 +307,7 @@ test_cipher_on_target() { # signed ones) local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ - -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + -untrusted <(printf "%s" "${current_raw_certificates[@]}") <(echo "$cert") 2>/dev/null | \ grep ': OK$' >/dev/null; then # if the certificate is an intermediate CA it may be useful @@ -354,7 +427,7 @@ get_cipher_pref() { if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") ciphercertificates=("${ciphercertificates[@]}" "$certificates") - pciph=($(echo $result)) + pciph=($result) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -371,30 +444,32 @@ display_results_in_terminal() { local different=False echo "Target: $TARGET"; echo for cipher in "${cipherspref[@]}"; do - pciph=($(echo $cipher)) + # get first in array + pciph=($cipher) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" else r="$ctr $cipher" fi + local cipher_data=($cipher) if [ $ctr -eq 1 ]; then - pubkey=$(awk '{print $3}' <<<$cipher) - sigalg=$(awk '{print $4}' <<<$cipher) - trusted=$(awk '{print $5}' <<<$cipher) - tickethint=$(awk '{print $6}' <<<$cipher) - ocspstaple=$(awk '{print $7}' <<<$cipher) + pubkey="${cipher_data[2]}" + sigalg="${cipher_data[3]}" + trusted="${cipher_data[4]}" + tickethint="${cipher_data[5]}" + ocspstaple="${cipher_data[6]}" else - if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then + if [ "$pubkey" != "${cipher_data[2]}" ]; then different=True fi - if [ "$sigalg" != "$(awk '{print $4}' <<<$cipher)" ]; then + if [ "$sigalg" != "${cipher_data[3]}" ]; then different=True fi - if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then + if [ "$trusted" != "${cipher_data[4]}" ]; then different=True fi - if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then + if [ "$tickethint" != "${cipher_data[5]}" ]; then different=True fi fi @@ -454,18 +529,19 @@ display_results_in_json() { ctr=0 echo -n "{\"target\":\"$TARGET\",\"utctimestamp\":\"$(date -u '+%FT%T.0Z')\",\"serverside\":\"${serverside}\",\"ciphersuite\": [" for cipher in "${cipherspref[@]}"; do + local cipher_arr=($cipher) [ $ctr -gt 0 ] && echo -n ',' - echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\"," - echo -n "\"protocols\":[\"$(echo $cipher|awk '{print $2}'|sed 's/,/","/g')\"]," - echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," - echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," - echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + echo -n "{\"cipher\":\"${cipher_arr[0]}\"," + echo -n "\"protocols\":[\"${cipher_arr[1]//,/\",\"}\"]," + echo -n "\"pubkey\":[\"${cipher_arr[2]//,/\",\"}\"]," + echo -n "\"sigalg\":[\"${cipher_arr[3]//,/\",\"}\"]," + echo -n "\"trusted\":\"${cipher_arr[4]//,/\",\"}\"," if [[ -n $CAPATH ]]; then echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," fi - echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," - echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," - pfs=$(echo $cipher|awk '{print $8}') + echo -n "\"ticket_hint\":\"${cipher_arr[5]}\"," + echo -n "\"ocsp_stapling\":\"${cipher_arr[6]}\"," + pfs="${cipher_arr[7]}" [ "$pfs" == "" ] && pfs="None" echo -n "\"pfs\":\"$pfs\"}" ctr=$((ctr+1)) @@ -483,25 +559,25 @@ test_serverside_ordering() { # server supports just two ciphers, so rotate them, that should be enough elif [[ ${#cipherspref[@]} -eq 2 ]]; then - local cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + local cipher=(${cipherspref[1]}) prefered="$cipher" ciphersuite=$cipher - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" # server supports 3 or more ciphers, rotate all three. This is necessary because google does # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or # ECDHE-RSA-CHACHA20-POLY1305 else - local cipher=$(awk '{print $1}' <<< ${cipherspref[2]}) + local cipher=(${cipherspref[2]}) prefered="$cipher" ciphersuite="$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + cipher=(${cipherspref[1]}) ciphersuite+=":$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" fi @@ -517,7 +593,7 @@ test_serverside_ordering() { if [ $? -ne 0 ]; then serverside="True" else - local selected=$(awk '{print $1}' <<< $result) + local selected=($result) if [[ $selected == $prefered ]]; then serverside="False" else @@ -614,8 +690,7 @@ TARGET=$HOST:$PORT debug "target: $TARGET" # test our openssl is usable -tmp="$($OPENSSLBIN -h 2>&1 1>/dev/null)" -if [ $? -gt 0 ]; then +if [ ! -x $OPENSSLBIN ]; then OPENSSLBIN=$(which openssl) if [ "$OUTPUTFORMAT" == "terminal" ]; then echo "custom openssl not executable, falling back to system one from $OPENSSLBIN"