From 512819a33f386b00a1f35d743ddc701b993883e7 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 13:42:48 +0200 Subject: [PATCH 01/10] no need to grep the input when we're using awk (v2) awk has an inbuilt version of grep, also truncate processing as soon as we find what we're looking for This version uses slightly different syntax that is compatible with old awk --- cipherscan | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cipherscan b/cipherscan index 7c02886..66dd47e 100755 --- a/cipherscan +++ b/cipherscan @@ -120,20 +120,21 @@ test_cipher_on_target() { tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") # session metadata - current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') - current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') - current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') - current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }') + current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") + current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") + current_protocol=$(awk '/^ +Protocol +:/ {print $3; exit}' <<<"$tmp") + current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") if [ -z $current_tickethint ]; then current_tickethint=None fi # certificate metadata - current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') + current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") if [ -z $current_pubkey ]; then current_pubkey=0 fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" + current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ + awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null if [ $? -eq 0 ]; then current_trusted="True" From 38f428b476f62ae0a2815dcc1a2c20137974f090 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 15:18:11 +0200 Subject: [PATCH 02/10] add caching of intermediate CA certificates --- cipherscan | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/cipherscan b/cipherscan index 66dd47e..04a206c 100755 --- a/cipherscan +++ b/cipherscan @@ -49,7 +49,7 @@ TIMEOUT=10 CAPATH="" usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -63,7 +63,8 @@ Use one of the options below: -a | --allciphers Test all known ciphers individually at the end. -b | --benchmark Activate benchmark mode. ---capath use CAs from directory +--capath use CAs from directory (must be in OpenSSL CAdir format) +--saveca save intermediate certificates in CA directory -d | --delay Pause for n seconds between connections -D | --debug Output ALL the information. -h | --help Shows this help text. @@ -91,6 +92,23 @@ debug(){ fi } +c_hash() { + local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) + for num in $(seq 0 100); do + if [[ $1/${h}.${num} -ef $2 ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $1/${h}.${num} ]]; then + # file doesn't exist, create a link + pushd "$1" > /dev/null + ln -s "$2" "${h}.${num}" + popd > /dev/null + break + fi + done +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -99,6 +117,7 @@ test_cipher_on_target() { protocols="" pfs="" previous_cipher="" + certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do # sslv2 client hello doesn't support SNI extension @@ -145,6 +164,63 @@ test_cipher_on_target() { current_sigalg=None fi + # collect certificate data + current_certificates="" + local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ + <<<"$tmp") + debug "server presented $certificate_count certificates" + local i + for ((i=0; i<$certificate_count; i=i+1 )); do + + # extract i'th certificate + local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {if (n == i) print } + ' <<<"$tmp") + + # compute sha256 fingerprint of the certificate + local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + + # check if it is a CA certificate + local isCA="False" + if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\ + grep 'CA:TRUE' >/dev/null; then + isCA="True" + fi + + # build trust source for certificate verification + local trust_source=() + if [[ -n $CAPATH ]]; then + trust_source=("-CApath" "$CAPATH") + elif [[ -e $CACERTS ]]; then + trust_source=("-CAfile" "$CACERTS") + fi + + # check if the certificate is actually trusted (server may present + # unrelated certificates that are not trusted (including self + # signed ones) + if ${OPENSSLBIN} verify "${trust_source[@]}" \ + -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + grep 'OK$' >/dev/null; then + + # if the certificate is an intermediate CA it may be useful + # for connecting to servers that are misconfigured so save it + if [[ -n $CAPATH ]] && [[ $SAVECA == "True" ]] && [[ $isCA == "True" ]]; then + if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then + echo "$cert" > "$CAPATH/${sha256sum}.pem" + c_hash "$CAPATH" "${sha256sum}.pem" + fi + fi + fi + # save the sha sum for reporting + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${sha256sum}\"" + done + debug "current_certificates: $current_certificates" + # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then # connection failed, try again with next TLS version @@ -171,6 +247,7 @@ test_cipher_on_target() { trusted=$current_trusted tickethint=$current_tickethint ocspstaple=$current_ocspstaple + certificates="$current_certificates" # grab the cipher and PFS key size done # if cipher is empty, that means none of the TLS version worked with @@ -222,7 +299,7 @@ get_cipher_pref() { local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e $CACERTS ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -234,6 +311,7 @@ get_cipher_pref() { # If the connection succeeded with the current cipher, benchmark and store if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") + ciphercertificates=("${ciphercertificates[@]}" "$certificates") pciph=$(echo $result|awk '{print $1}') get_cipher_pref "!$pciph:$ciphersuite" return 0 @@ -340,6 +418,9 @@ display_results_in_json() { echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + if [[ -n $CAPATH ]]; then + echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," + fi echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," pfs=$(echo $cipher|awk '{print $8}') @@ -384,7 +465,7 @@ test_serverside_ordering() { local sslcommand="$TIMEOUTBIN $TIMEOUT $OPENSSLBIN s_client" if [ -n "$CAPATH" ]; then - sslcommand+=" -CApath $CAPATH" + sslcommand+=" -CApath $CAPATH -showcerts" elif [ -e "$CACERTS" ]; then sslcommand+=" -CAfile $CACERTS" fi @@ -444,6 +525,10 @@ do CAPATH="$2" shift 2 ;; + --saveca) + SAVECA="True" + shift 1 + ;; --) # End of all options shift break @@ -496,6 +581,7 @@ debug "sclientargs: $SCLIENTARGS" cipherspref=(); +ciphercertificates=() results=() # Call to the recursive loop that retrieves the cipher preferences From eaa64e48a3f1b11065e39cce5d758d406a7bf895 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 15:18:48 +0200 Subject: [PATCH 03/10] add ability to also save leaf certificates and untrusted ones --- cipherscan | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index 04a206c..02838f7 100755 --- a/cipherscan +++ b/cipherscan @@ -47,9 +47,10 @@ TIMEOUT=10 # place where to put the found intermediate CA certificates and where # trust anchors are stored CAPATH="" +SAVECRT="" usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -70,6 +71,7 @@ Use one of the options below: -h | --help Shows this help text. -j | --json Output results in JSON format. -o | --openssl path/to/your/openssl binary you want to use. +--savecrt path where to save untrusted and leaf certificates -v | --verbose Increase verbosity. The rest of the arguments will be interpreted as openssl s_client argument. @@ -200,6 +202,7 @@ test_cipher_on_target() { # check if the certificate is actually trusted (server may present # unrelated certificates that are not trusted (including self # signed ones) + local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ grep 'OK$' >/dev/null; then @@ -211,6 +214,12 @@ test_cipher_on_target() { echo "$cert" > "$CAPATH/${sha256sum}.pem" c_hash "$CAPATH" "${sha256sum}.pem" fi + saved="True" + fi + fi + if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then + if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then + echo "$cert" > "$SAVECRT/${sha256sum}.pem" fi fi # save the sha sum for reporting @@ -529,6 +538,10 @@ do SAVECA="True" shift 1 ;; + --savecrt) + SAVECRT="$2" + shift 2 + ;; --) # End of all options shift break From ea9f0056eb30c7ca9c67563b782102d84c8078da Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 11 Jul 2014 17:29:57 +0200 Subject: [PATCH 04/10] clean up the extracted certificate the certificate extracted in the above way will contain some junk from openssl s_client output we don't want like verification status we can remove it ro reduce disk usage for saved certificates --- cipherscan | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cipherscan b/cipherscan index 02838f7..3a253c5 100755 --- a/cipherscan +++ b/cipherscan @@ -179,6 +179,8 @@ test_cipher_on_target() { /-----END CERTIFICATE-----/ {split_after=1} {if (n == i) print } ' <<<"$tmp") + # clean up the cert from junk before BEGIN CERTIFICATE + cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) # compute sha256 fingerprint of the certificate local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ From c2b3b6d7aa5fcfff7abc52a1ec2b02abb6d97c28 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 11 Oct 2014 14:15:59 +0200 Subject: [PATCH 05/10] use CApath for certificates and store certificates (v2) CApath is about 20% faster than CAfile so use it, also save the received certificates from the servers for later analysis (proper hostname checking, looking for certificates sharing private key, etc.) Use the mechanism from cipherscan to find location of ca cert bundle --- top1m/testtop1m.sh | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index e77c1e1..23dbbc1 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -9,6 +9,42 @@ if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then exit 1 fi [ ! -e "results" ] && mkdir results +[ ! -e "certs" ] && mkdir certs +if [ -z "$CACERTS" ]; then + for f in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do + if [ -e "$f" ]; then + CACERTS="$f" + break + fi + done +fi +if [ ! -e "$CACERTS" ]; then + echo "file with CA certificates does not exist, please export CACERTS variable with location" + exit 1 +fi +if [ ! -e "ca_files" ]; then + mkdir ca_files + pushd ca_files >/dev/null + awk ' + split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {print > "cert" n ".pem"}' < "$CACERTS" + for i in *; do + h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null) + for num in `seq 0 100`; do + if [[ $h.$num -ef $i ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $h.$num ]]; then + # file doesn't exist, create a link + ln -s "$i" "$h.$num" + break + fi + done + done + popd >/dev/null +fi function wait_for_jobs() { local no_jobs @@ -32,7 +68,7 @@ function scan_host() { if [ $? -gt 0 ]; then return fi - ../cipherscan -json -servername $1 $2:443 > results/$1@$2 + ../cipherscan --capath ca_files --saveca --savecrt certs -json -servername $1 $2:443 > results/$1@$2 } function scan_host_no_sni() { @@ -44,7 +80,7 @@ function scan_host_no_sni() { if [ $? -gt 0 ]; then return fi - ../cipherscan -json $1:443 > results/$1 + ../cipherscan --capath ca_files --saveca --savecrt certs -json $1:443 > results/$1 } function scan_hostname() { From 9f9af9c71df24782498515c475955aaa78f366f5 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:17:52 +0200 Subject: [PATCH 06/10] don't calculate sha sums for the certificates over and over we can use cksum to calculate simple checksum much faster than with using openssl, so we can compute sums only once --- cipherscan | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cipherscan b/cipherscan index 3a253c5..3d39fca 100755 --- a/cipherscan +++ b/cipherscan @@ -48,6 +48,10 @@ TIMEOUT=10 # trust anchors are stored CAPATH="" SAVECRT="" +unset known_certs +declare -A known_certs +unset cert_checksums +declare -A cert_checksums usage() { echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] @@ -175,15 +179,25 @@ test_cipher_on_target() { for ((i=0; i<$certificate_count; i=i+1 )); do # extract i'th certificate - local cert=$(awk -v i=$i 'split_after == 1 {n++;split_after=0} - /-----END CERTIFICATE-----/ {split_after=1} - {if (n == i) print } - ' <<<"$tmp") - # clean up the cert from junk before BEGIN CERTIFICATE - cert=$(${OPENSSLBIN} x509 <<<"$cert" 2>/dev/null) + local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } + /-----BEGIN CERTIFICATE-----/ { output=1 } + output==1 { if (n==i) print } + /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + # put the output to an array instead awk '{print $1}' + local cksum=($(cksum <<<"$cert")) + # compare the values not just checksums so that eventual collision + # doesn't mess up results + if [[ ${known_certs[$cksum]} == $cert ]]; then + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${cert_checksums[$cksum]}\"" + continue + fi # compute sha256 fingerprint of the certificate - local sha256sum=$(${OPENSSLBIN} x509 -outform DER <<<"$cert" 2>/dev/null |\ + local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + <<<"$cert" 2>/dev/null |\ ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') # check if it is a CA certificate @@ -229,6 +243,8 @@ test_cipher_on_target() { current_certificates+="," fi current_certificates+="\"${sha256sum}\"" + known_certs[$cksum]="$cert" + cert_checksums[$cksum]="$sha256sum" done debug "current_certificates: $current_certificates" From 0071870c7c69bb15541002ded2fb458e74b4c00b Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 14:54:33 +0200 Subject: [PATCH 07/10] few less forks in the script again, we can use arrays and a bit advanced awk syntax to reduce the number of forks necessary to run the script --- cipherscan | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cipherscan b/cipherscan index 3d39fca..f247bbd 100755 --- a/cipherscan +++ b/cipherscan @@ -183,7 +183,7 @@ test_cipher_on_target() { /-----BEGIN CERTIFICATE-----/ { output=1 } output==1 { if (n==i) print } /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") - # put the output to an array instead awk '{print $1}' + # put the output to an array instead running awk '{print $1}' local cksum=($(cksum <<<"$cert")) # compare the values not just checksums so that eventual collision # doesn't mess up results @@ -339,7 +339,7 @@ get_cipher_pref() { if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") ciphercertificates=("${ciphercertificates[@]}" "$certificates") - pciph=$(echo $result|awk '{print $1}') + pciph=($(echo $result)) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -356,7 +356,7 @@ display_results_in_terminal() { local ocspstaple local different=False for cipher in "${cipherspref[@]}"; do - pciph=$(echo $cipher|awk '{print $1}') + pciph=($(echo $cipher)) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" @@ -410,7 +410,7 @@ display_results_in_terminal() { echo $result|grep -v '(NONE)' else # prints priority, ciphersuite, protocols and pfs_keysize - echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}' + awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result" fi done|column -t echo From 0e6c40ad83db05fd78cb99dc27492cbdc55f41c1 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 4 Aug 2014 17:22:53 +0200 Subject: [PATCH 08/10] make handling of self signed certs more robust openssl sometimes will print the filename, then the error, and finish with OK, matching the colon and space prevents from considering such certs to be valid --- cipherscan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipherscan b/cipherscan index f247bbd..d1ff13f 100755 --- a/cipherscan +++ b/cipherscan @@ -221,7 +221,7 @@ test_cipher_on_target() { local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ - grep 'OK$' >/dev/null; then + grep ': OK$' >/dev/null; then # if the certificate is an intermediate CA it may be useful # for connecting to servers that are misconfigured so save it From 2e9c3fcc90f0d008a6c5cf7429c7649a9d96c987 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 12 Jul 2014 01:30:17 +0200 Subject: [PATCH 09/10] don't retry protocols we know don't work When connection is unsuccessful with a given protocol, don't try it again since we probably exhausted the ciphers supported by the protocol makes scanning about 10% faster --- cipherscan | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cipherscan b/cipherscan index d1ff13f..016c5a4 100755 --- a/cipherscan +++ b/cipherscan @@ -52,6 +52,8 @@ unset known_certs declare -A known_certs unset cert_checksums declare -A cert_checksums +unset ok_protocols +declare -A ok_protocols usage() { echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] @@ -126,6 +128,9 @@ test_cipher_on_target() { certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do + if [[ ${ok_protocols[$tls_version]} -eq 1 ]]; then + continue + fi # sslv2 client hello doesn't support SNI extension # in SSLv3 mode OpenSSL just ignores the setting so it's ok # -status exception is ignored in SSLv2, go figure @@ -250,6 +255,7 @@ test_cipher_on_target() { # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then + ok_protocols["$tls_version"]=1 # connection failed, try again with next TLS version continue else @@ -617,6 +623,8 @@ results=() # Call to the recursive loop that retrieves the cipher preferences get_cipher_pref $CIPHERSUITE +unset ok_protocols +declare -A ok_protocols test_serverside_ordering From 5b96f8fb4726827f2e937a5c72cd516ec757a75a Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 18 Oct 2014 17:20:20 +0200 Subject: [PATCH 10/10] limit number of forks needed to speed up execution bash has a built in regular expression processor, we can match lines using =~ moreover, stuff that will match while being inside parentheses is later available in the BASH_REMATCH array the IFS (Internal Field Separator) by default includes space, tab and new line, as such we can use it to split longer lines to separate words, just as awk '{print $1}' can, just need to put the value to an array for that we also don't have to use $(echo $var) when assigning variables, $var is enough bash has also built in substitution engine, so we can do ${var/,/ & } to switch all commas to ampersands when using the variable --- cipherscan | 217 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 146 insertions(+), 71 deletions(-) diff --git a/cipherscan b/cipherscan index 016c5a4..233b655 100755 --- a/cipherscan +++ b/cipherscan @@ -56,7 +56,9 @@ unset ok_protocols declare -A ok_protocols usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] +[--saveca] [--savecrt directory] [-d|--delay seconds] [-D|--debug] [-j|--json] +[-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -102,7 +104,7 @@ debug(){ c_hash() { local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) - for num in $(seq 0 100); do + for ((num=0; num<=100; num++)) ; do if [[ $1/${h}.${num} -ef $2 ]]; then # file already linked, ignore break @@ -117,6 +119,106 @@ c_hash() { done } +parse_openssl_output() { + # clear variables in case matching doesn't hit them + current_ocspstaple="False" + current_cipher="" + current_pfs="" + current_protocol="" + current_tickethint="None" + current_pubkey=0 + current_trusted="False" + current_sigalg="None" + + certs_found=0 + current_raw_certificates=() + + while read line; do + # check if there isn't OCSP response data (response and responder cert) + if [[ $line =~ ^====================================== ]]; then + while read data; do + # check if there is a OCSP response in output + if [[ $data =~ OCSP\ Response\ Data ]]; then + current_ocspstaple="True" + continue + fi + + # skip all data from a OCSP response + if [[ $data =~ ^====================================== ]]; then + break + fi + done + continue + fi + + # extract selected cipher + if [[ $line =~ New,\ ]]; then + local match=($line) + current_cipher="${match[4]}" + continue + fi + + # extract data about selected temporary key + if [[ $line =~ Server\ Temp\ Key ]]; then + local match=($line) + current_pfs="${match[3]}${match[4]}${match[5]}${match[6]}" + continue + fi + + # extract used protocol + if [[ $line =~ ^Protocol\ + ]]; then + local match=($line) + current_protocol="${match[2]}" + continue + fi + + # extract session ticket hint + if [[ $line =~ ticket\ lifetime\ hint ]]; then + local match=($line) + current_tickethint="${match[5]}" + continue + fi + + # extract size of server public key + if [[ $line =~ Server\ public\ key\ is\ ]]; then + local match=($line) + current_pubkey="${match[4]}" + continue + fi + + # check if connection used trused certificate + if [[ $line =~ Verify\ return\ code:\ 0 ]]; then + current_trusted="True" + continue + fi + + # extract certificates + if [[ $line =~ -----BEGIN\ CERTIFICATE----- ]]; then + current_raw_certificates[$certs_found]="$line"$'\n' + while read data; do + current_raw_certificates[$certs_found]+="$data"$'\n' + if [[ $data =~ -----END\ CERTIFICATE----- ]]; then + break + fi + done + certs_found=$((certs_found+1)) + continue + fi + done + + # if we found any certs in output, process the first one and extract + # the signature algorithm on it (it's the server's certificate) + if [[ $certs_found -gt 0 ]]; then + local ossl_out=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"${current_raw_certificates[0]}") + while read data; do + if [[ $data =~ Signature\ Algorithm ]]; then + local match=($data) + current_sigalg="${match[2]}" + fi + done <<<"$ossl_out" + fi +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ @@ -135,59 +237,30 @@ test_cipher_on_target() { # in SSLv3 mode OpenSSL just ignores the setting so it's ok # -status exception is ignored in SSLv2, go figure if [ "$tls_version" == "-ssl2" ]; then - cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand) + if [[ "$sslcommand" =~ (.*)(-servername\ [^ ]*)(.*) ]]; then + cmnd="${BASH_REMATCH[1]} ${BASH_REMATCH[3]}" + else + cmnd="$sslcommand" + fi else cmnd=$sslcommand fi debug echo \"Q\" \| $cmnd $tls_version local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null) - if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then - current_ocspstaple="True" - else - current_ocspstaple="False" - fi - # filter out the OCSP server certificate - tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") - # session metadata - current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") - current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") - current_protocol=$(awk '/^ +Protocol +:/ {print $3; exit}' <<<"$tmp") - current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") - if [ -z $current_tickethint ]; then - current_tickethint=None - fi - - # certificate metadata - current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") - if [ -z $current_pubkey ]; then - current_pubkey=0 - fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ - awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" - grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null - if [ $? -eq 0 ]; then - current_trusted="True" - else - current_trusted="False" - fi - if [ -z $current_sigalg ]; then - current_sigalg=None - fi + parse_openssl_output <<<"$tmp" + verbose "selected cipher is '$current_cipher'" + verbose "using protocol '$current_protocol'" # collect certificate data current_certificates="" - local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ - <<<"$tmp") + local certificate_count=$certs_found debug "server presented $certificate_count certificates" local i for ((i=0; i<$certificate_count; i=i+1 )); do # extract i'th certificate - local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } - /-----BEGIN CERTIFICATE-----/ { output=1 } - output==1 { if (n==i) print } - /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + local cert="${current_raw_certificates[$i]}" # put the output to an array instead running awk '{print $1}' local cksum=($(cksum <<<"$cert")) # compare the values not just checksums so that eventual collision @@ -201,9 +274,9 @@ test_cipher_on_target() { fi # compute sha256 fingerprint of the certificate - local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + local sha256sum=($(${OPENSSLBIN} x509 -outform DER\ <<<"$cert" 2>/dev/null |\ - ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null)) # check if it is a CA certificate local isCA="False" @@ -225,7 +298,7 @@ test_cipher_on_target() { # signed ones) local saved="False" if ${OPENSSLBIN} verify "${trust_source[@]}" \ - -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + -untrusted <(printf "%s" "${current_raw_certificates[@]}") <(echo "$cert") 2>/dev/null | \ grep ': OK$' >/dev/null; then # if the certificate is an intermediate CA it may be useful @@ -345,7 +418,7 @@ get_cipher_pref() { if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") ciphercertificates=("${ciphercertificates[@]}" "$certificates") - pciph=($(echo $result)) + pciph=($result) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -362,30 +435,32 @@ display_results_in_terminal() { local ocspstaple local different=False for cipher in "${cipherspref[@]}"; do - pciph=($(echo $cipher)) + # get first in array + pciph=($cipher) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" else r="$ctr $cipher" fi + local cipher_data=($cipher) if [ $ctr -eq 1 ]; then - pubkey=$(awk '{print $3}' <<<$cipher) - sigalg=$(awk '{print $4}' <<<$cipher) - trusted=$(awk '{print $5}' <<<$cipher) - tickethint=$(awk '{print $6}' <<<$cipher) - ocspstaple=$(awk '{print $7}' <<<$cipher) + pubkey="${cipher_data[2]}" + sigalg="${cipher_data[3]}" + trusted="${cipher_data[4]}" + tickethint="${cipher_data[5]}" + ocspstaple="${cipher_data[6]}" else - if [ "$pubkey" != "$(awk '{print $3}' <<<$cipher)" ]; then + if [ "$pubkey" != "${cipher_data[2]}" ]; then different=True fi - if [ "$sigalg" != "$(awk '{print $4}' <<<$cipher)" ]; then + if [ "$sigalg" != "${cipher_data[3]}" ]; then different=True fi - if [ "$trusted" != "$(awk '{print $5}' <<<$cipher)" ]; then + if [ "$trusted" != "${cipher_data[4]}" ]; then different=True fi - if [ "$tickethint" != "$(awk '{print $6}' <<<$cipher)" ]; then + if [ "$tickethint" != "${cipher_data[5]}" ]; then different=True fi fi @@ -445,18 +520,19 @@ display_results_in_json() { ctr=0 echo -n "{\"target\":\"$TARGET\",\"utctimestamp\":\"$(date -u '+%FT%T.0Z')\",\"serverside\":\"${serverside}\",\"ciphersuite\": [" for cipher in "${cipherspref[@]}"; do + local cipher_arr=($cipher) [ $ctr -gt 0 ] && echo -n ',' - echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\"," - echo -n "\"protocols\":[\"$(echo $cipher|awk '{print $2}'|sed 's/,/","/g')\"]," - echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," - echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," - echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + echo -n "{\"cipher\":\"${cipher_arr[0]}\"," + echo -n "\"protocols\":[\"${cipher_arr[1]//,/\",\"}\"]," + echo -n "\"pubkey\":[\"${cipher_arr[2]//,/\",\"}\"]," + echo -n "\"sigalg\":[\"${cipher_arr[3]//,/\",\"}\"]," + echo -n "\"trusted\":\"${cipher_arr[4]//,/\",\"}\"," if [[ -n $CAPATH ]]; then echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," fi - echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," - echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," - pfs=$(echo $cipher|awk '{print $8}') + echo -n "\"ticket_hint\":\"${cipher_arr[5]}\"," + echo -n "\"ocsp_stapling\":\"${cipher_arr[6]}\"," + pfs="${cipher_arr[7]}" [ "$pfs" == "" ] && pfs="None" echo -n "\"pfs\":\"$pfs\"}" ctr=$((ctr+1)) @@ -474,25 +550,25 @@ test_serverside_ordering() { # server supports just two ciphers, so rotate them, that should be enough elif [[ ${#cipherspref[@]} -eq 2 ]]; then - local cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + local cipher=(${cipherspref[1]}) prefered="$cipher" ciphersuite=$cipher - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" # server supports 3 or more ciphers, rotate all three. This is necessary because google does # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or # ECDHE-RSA-CHACHA20-POLY1305 else - local cipher=$(awk '{print $1}' <<< ${cipherspref[2]}) + local cipher=(${cipherspref[2]}) prefered="$cipher" ciphersuite="$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + cipher=(${cipherspref[1]}) ciphersuite+=":$cipher" - cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + cipher=(${cipherspref[0]}) ciphersuite+=":$cipher" fi @@ -508,7 +584,7 @@ test_serverside_ordering() { if [ $? -ne 0 ]; then serverside="True" else - local selected=$(awk '{print $1}' <<< $result) + local selected=($result) if [[ $selected == $prefered ]]; then serverside="False" else @@ -605,8 +681,7 @@ TARGET=$HOST:$PORT debug "target: $TARGET" # test our openssl is usable -tmp="$($OPENSSLBIN -h 2>&1 1>/dev/null)" -if [ $? -gt 0 ]; then +if [ ! -x $OPENSSLBIN ]; then OPENSSLBIN=$(which openssl) if [ "$OUTPUTFORMAT" == "terminal" ]; then echo "custom openssl not executable, falling back to system one from $OPENSSLBIN"