diff --git a/cipherscan b/cipherscan index eb91255..2f3b024 100755 --- a/cipherscan +++ b/cipherscan @@ -22,6 +22,8 @@ fi if [ ! -e "$CACERTS" ]; then echo "Warning: CA Certificates not found at $CACERTS, export CACERTS variable with location of your trust anchors" 1>&2 fi +# RSA ciphers are put at the end to force Google servers to accept ECDSA ciphers +# (probably a result of a workaround for the bug in Apple implementation of ECDSA) CIPHERSUITE="ALL:COMPLEMENTOFALL:+aRSA" DEBUG=0 VERBOSE=0 @@ -29,10 +31,19 @@ DELAY=0 ALLCIPHERS=0 OUTPUTFORMAT="terminal" TIMEOUT=10 - +# place where to put the found intermediate CA certificates and where +# trust anchors are stored +CAPATH="" +SAVECRT="" +unset ok_protocols +declare -A ok_protocols +unset known_certs +declare -A known_certs +unset cert_checksums +declare -A cert_checksums usage() { - echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] + echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [--savecrt directory] [-v|--verbose] [-o|--openssl file] [openssl s_client args] usage: $0 -h|--help $0 attempts to connect to a target site using all the ciphersuites it knows. @@ -46,11 +57,13 @@ Use one of the options below: -a | --allciphers Test all known ciphers individually at the end. -b | --benchmark Activate benchmark mode. +--capath use CAs from directory, save intermediate certificates there -d | --delay Pause for n seconds between connections -D | --debug Output ALL the information. -h | --help Shows this help text. -j | --json Output results in JSON format. -o | --openssl path/to/your/openssl binary you want to use. +--savecrt path where to save untrusted and leaf certificates -v | --verbose Increase verbosity. The rest of the arguments will be interpreted as openssl s_client argument. @@ -74,17 +87,47 @@ debug(){ fi } +c_hash() { + local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null) + for num in $(seq 0 100); do + if [[ $1/${h}.${num} -ef $2 ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $1/${h}.${num} ]]; then + # file doesn't exist, create a link + pushd "$1" > /dev/null + ln -s "$2" "${h}.${num}" + popd > /dev/null + break + fi + done +} + # Connect to a target host with the selected ciphersuite test_cipher_on_target() { local sslcommand=$@ cipher="" + local cmnd="" protocols="" pfs="" previous_cipher="" + certificates="" for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2" do - debug echo \"Q\" \| $sslcommand $tls_version - local tmp=$(echo "Q" | $sslcommand $tls_version 1>/dev/stdout 2>/dev/null) + if [[ ${ok_protocols[$tls_version]} -eq 1 ]]; then + continue + fi + # sslv2 client hello doesn't support SNI extension + # in SSLv3 mode OpenSSL just ignores the setting so it's ok + # -status exception is ignored in SSLv2, go figure + if [ "$tls_version" == "-ssl2" ]; then + cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand) + else + cmnd=$sslcommand + fi + debug echo \"Q\" \| $cmnd $tls_version + local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null) if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then current_ocspstaple="True" else @@ -92,18 +135,23 @@ test_cipher_on_target() { fi # filter out the OCSP server certificate tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp") - current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}') - current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}') - current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}') - current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}') - if [ -z $current_pubkey ]; then - current_pubkey=0 - fi - current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }') + + # session metadata + current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp") + current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp") + current_protocol=$(awk '/^\s+Protocol\s+:/ {print $3; exit}' <<<"$tmp") + current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp") if [ -z $current_tickethint ]; then current_tickethint=None fi - current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None" + + # certificate metadata + current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp") + if [ -z $current_pubkey ]; then + current_pubkey=0 + fi + current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\ + awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None" grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null if [ $? -eq 0 ]; then current_trusted="True" @@ -113,7 +161,88 @@ test_cipher_on_target() { if [ -z $current_sigalg ]; then current_sigalg=None fi + + # collect certificate data + current_certificates="" + local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\ + <<<"$tmp") + debug "server presented $certificate_count certificates" + local i + for ((i=0; i<$certificate_count; i=i+1 )); do + + # extract i'th certificate + local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 } + /-----BEGIN CERTIFICATE-----/ { output=1 } + output==1 { if (n==i) print } + /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp") + # put the output to an array instead running awk '{print $1}' + local cksum=($(cksum <<<"$cert")) + # compare the values not just checksums so that eventual collision + # doesn't mess up results + if [[ ${known_certs[$cksum]} == $cert ]]; then + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${cert_checksums[$cksum]}\"" + continue + fi + + # compute sha256 fingerprint of the certificate + local sha256sum=$(${OPENSSLBIN} x509 -outform DER\ + <<<"$cert" 2>/dev/null |\ + ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}') + + # check if it is a CA certificate + local isCA="False" + if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\ + grep 'CA:TRUE' >/dev/null; then + isCA="True" + fi + + # build trust source for certificate verification + local trust_source=() + if [[ -n $CAPATH ]]; then + trust_source=("-CApath" "$CAPATH") + elif [[ -e $CACERTS ]]; then + trust_source=("-CAfile" "$CACERTS") + fi + + # check if the certificate is actually trusted (server may present + # unrelated certificates that are not trusted (including self + # signed ones) + local saved="False" + if ${OPENSSLBIN} verify "${trust_source[@]}" \ + -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \ + grep ': OK$' >/dev/null; then + + # if the certificate is an intermediate CA it may be useful + # for connecting to servers that are misconfigured so save it + if [[ -n $CAPATH ]] && [[ $isCA == "True" ]]; then + if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then + echo "$cert" > "$CAPATH/${sha256sum}.pem" + c_hash "$CAPATH" "${sha256sum}.pem" + fi + saved="True" + fi + fi + if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then + if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then + echo "$cert" > "$SAVECRT/${sha256sum}.pem" + fi + fi + # save the sha sum for reporting + if [ -n "${current_certificates}" ]; then + current_certificates+="," + fi + current_certificates+="\"${sha256sum}\"" + known_certs[$cksum]="$cert" + cert_checksums[$cksum]="$sha256sum" + done + debug "current_certificates: $current_certificates" + + # parsing finished, report result if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then + ok_protocols["$tls_version"]=1 # connection failed, try again with next TLS version continue else @@ -138,6 +267,7 @@ test_cipher_on_target() { trusted=$current_trusted tickethint=$current_tickethint ocspstaple=$current_ocspstaple + certificates="$current_certificates" # grab the cipher and PFS key size done # if cipher is empty, that means none of the TLS version worked with @@ -182,24 +312,28 @@ bench_cipher() { cipherbenchms="$((t/1000/$BENCHMARKITER))" } - # Connect to the target and retrieve the chosen cipher # recursively until the connection fails get_cipher_pref() { [ "$OUTPUTFORMAT" == "terminal" ] && [ $DEBUG -lt 1 ] && echo -n '.' local ciphersuite="$1" - if [ -e $CACERTS ]; then - local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" - else - local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + + local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client" + if [ -n "$CAPATH" ]; then + sslcommand+=" -CApath $CAPATH -showcerts" + elif [ -e $CACERTS ]; then + sslcommand+=" -CAfile $CACERTS" fi + sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'" test_cipher_on_target "$sslcommand" local success=$? # If the connection succeeded with the current cipher, benchmark and store if [ $success -eq 0 ]; then cipherspref=("${cipherspref[@]}" "$result") - pciph=$(echo $result|awk '{print $1}') + ciphercertificates=("${ciphercertificates[@]}" "$certificates") + pciph=($(echo $result)) get_cipher_pref "!$pciph:$ciphersuite" return 0 fi @@ -217,7 +351,7 @@ display_results_in_terminal() { local ocspstaple local different=False for cipher in "${cipherspref[@]}"; do - pciph=$(echo $cipher|awk '{print $1}') + pciph=($(echo $cipher)) if [ $DOBENCHMARK -eq 1 ]; then bench_cipher "$pciph" r="$ctr $cipher $cipherbenchms" @@ -250,13 +384,13 @@ display_results_in_terminal() { if [ $DOBENCHMARK -eq 1 ]; then if [ $different == "True" ]; then - header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint pfs_keysize avg_handshake_microsec" + header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint ocsp_staple pfs_keysize avg_handshake_microsec" else header="prio ciphersuite protocols pfs_keysize avg_handshake_microsec" fi else if [ $different == "True" ]; then - header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint pfs_keysize" + header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint ocsp_staple pfs_keysize" else header="prio ciphersuite protocols pfs_keysize" fi @@ -268,10 +402,14 @@ display_results_in_terminal() { ctr=$((ctr+1)) fi if [ $different == "True" ]; then - echo $result|grep -v '(NONE)' + if [[ $(awk '{print $3}' <<< $result) == "SSLv3,TLSv1,TLSv1.1,TLSv1.2" ]]; then + awk '!/(NONE)/{print $1 " " $2 " " "SSLv3-TLSv1.2" " " $4 " " $5 " " $6 " " $7 " " $8 " " $9}' <<<"$result" + else + echo $result|grep -v '(NONE)' + fi else # prints priority, ciphersuite, protocols and pfs_keysize - echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}' + awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result" fi done|column -t echo @@ -288,13 +426,18 @@ display_results_in_terminal() { else echo "OCSP stapling: not supported" fi + if [[ $serverside == "True" ]]; then + echo "Server side cipher ordering" + else + echo "Client side cipher ordering" + fi } display_results_in_json() { # Display the results in json ctr=0 - echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"ciphersuite\": [" + echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"serverside\":\"${serverside}\",\"ciphersuite\": [" for cipher in "${cipherspref[@]}"; do [ $ctr -gt 0 ] && echo -n ',' echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\"," @@ -302,6 +445,9 @@ display_results_in_json() { echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"]," echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"]," echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\"," + if [[ -n $CAPATH ]]; then + echo -n "\"certificates\":[${ciphercertificates[$ctr]}]," + fi echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\"," echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\"," pfs=$(echo $cipher|awk '{print $8}') @@ -312,6 +458,60 @@ display_results_in_json() { echo ']}' } +test_serverside_ordering() { + + local ciphersuite="" + local prefered="" + # server supports only one cipher or no ciphers, so it effectively uses server side ordering... + if [[ ${#cipherspref[@]} -lt 2 ]]; then + serverside="True" + return 0 + # server supports just two ciphers, so rotate them, that should be enough + elif [[ ${#cipherspref[@]} -eq 2 ]]; then + + local cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + prefered="$cipher" + ciphersuite=$cipher + + cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + ciphersuite+=":$cipher" + + # server supports 3 or more ciphers, rotate all three. This is necessary because google does + # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or + # ECDHE-RSA-CHACHA20-POLY1305 + else + local cipher=$(awk '{print $1}' <<< ${cipherspref[2]}) + prefered="$cipher" + ciphersuite="$cipher" + + cipher=$(awk '{print $1}' <<< ${cipherspref[1]}) + ciphersuite+=":$cipher" + + cipher=$(awk '{print $1}' <<< ${cipherspref[0]}) + ciphersuite+=":$cipher" + fi + + local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client" + if [ -n "$CAPATH" ]; then + sslcommand+=" -CApath $CAPATH -showcerts" + elif [ -e "$CACERTS" ]; then + sslcommand+=" -CAfile $CACERTS" + fi + sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite" + + test_cipher_on_target "$sslcommand" + if [ $? -ne 0 ]; then + serverside="True" + else + local selected=$(awk '{print $1}' <<< $result) + if [[ $selected == $prefered ]]; then + serverside="False" + else + serverside="True" + fi + fi +} + # UNKNOWNOPTIONS="" while : do @@ -349,6 +549,14 @@ do DELAY=$2 shift 2 ;; + --capath) + CAPATH="$2" + shift 2 + ;; + --savecrt) + SAVECRT="$2" + shift 2 + ;; --) # End of all options shift break @@ -393,10 +601,15 @@ debug "sclientargs: $SCLIENTARGS" cipherspref=(); +ciphercertificates=() results=() # Call to the recursive loop that retrieves the cipher preferences get_cipher_pref $CIPHERSUITE +unset ok_protocols +declare -A ok_protocols + +test_serverside_ordering if [ "$OUTPUTFORMAT" == "json" ]; then display_results_in_json diff --git a/top1m/parse_CAs.c b/top1m/parse_CAs.c new file mode 100644 index 0000000..0b15609 --- /dev/null +++ b/top1m/parse_CAs.c @@ -0,0 +1,509 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * Author: Hubert Kario - 2014 + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static char* CA_TRUSTED = "./ca_trusted"; +static char* CA_ALL = "./ca_files"; +static char* CERTS_DIR = "./certs"; + +/* SSL context that knows only about trust anchors */ +SSL_CTX *trusted_only; +/* SSL context that also has access to other CA certs */ +SSL_CTX *all_CAs; + +// load certificate from file to a OpenSSL object +X509 *load_cert(char *filename) +{ + BIO* f; + X509 *ret; + + f = BIO_new(BIO_s_file()); + BIO_read_filename(f, filename); + + ret = PEM_read_bio_X509_AUX(f, NULL, 0, NULL); + if (ret == NULL) + fprintf(stderr, "Unable to load file %s as X509 certificate\n", filename); + + BIO_free_all(f); + + return ret; +} + +// convert sha256 to a file name, if the file exists +// search in "all CAs" dir and "leaf certs" directories +char *hash_to_filename(const char *hash) +{ + char *tmp_f_name; + size_t n; + + n = strlen(hash) + 30; + + // TODO error checking + tmp_f_name = malloc(n); + + /* first check if the file is in directory with regular certs */ + // TODO error checking + snprintf(tmp_f_name, n, "%s/%s.pem", CERTS_DIR, hash); + if (access(tmp_f_name, F_OK) != -1) { + return tmp_f_name; + } + + snprintf(tmp_f_name, n, "%s/%s.pem", CA_ALL, hash); + if (access(tmp_f_name, F_OK) != -1) { + return tmp_f_name; + } + + // file not found + free(tmp_f_name); + return NULL; +} + +// take certificate hashes, check their validity and output json that +// will indicate which certificate were used for verification, whatever +// the chain was trusted and if all certificates needed for verification +// (with the exception of root CA) were present in hashes +int process_chain(const char **cert_hashes) +{ + int ret; + int rc; // return code from function + char *f_name; + + X509 *cert; + X509 *x509; + + X509_STORE *store; + + X509_STORE_CTX *csc; + + STACK_OF(X509) *ustack; + STACK_OF(X509) *vstack; + + // load certificates to temp structures + + // first the end entity cert + // (EE cert needs to be passed separately to OpenSSL verification context) + f_name = hash_to_filename(cert_hashes[0]); + if (f_name == NULL) + return 1; + + cert = load_cert(f_name); + free(f_name); + if (cert == NULL) { + printf("can't load certificate!\n"); + return 1; + } + + // then the intermediate certificates + ustack = sk_X509_new_null(); + + for (int i=1; cert_hashes[i]!=NULL; i++) { + //printf(".\n"); + f_name = hash_to_filename(cert_hashes[i]); + if (f_name == NULL) { + // file not found + continue; + } + x509 = load_cert(f_name); + if (x509 == NULL) { + // loading cert failed + continue; + } + sk_X509_push(ustack, x509); + free(f_name); + } + + // first try with just trusted certificates + + store = SSL_CTX_get_cert_store(trusted_only); + if (store == NULL) { + fprintf(stderr, "store init failed\n"); + return 1; + } + X509_STORE_set_flags(store, X509_V_FLAG_TRUSTED_FIRST); + + csc = X509_STORE_CTX_new(); + + ret = X509_STORE_CTX_init(csc, store, cert, ustack); + if (ret != 1) { + return 1; + } + + ret = X509_verify_cert(csc); + + if (ret != 1) { + // printf("%s\n", X509_verify_cert_error_string(csc->error)); + } else { + // chain is complete, output certificate hashes + printf("{\"chain\":\"complete\",\"certificates\":["); + vstack = X509_STORE_CTX_get_chain(csc); + for(int i=0; i= 0) { + lseek(fd, -1, SEEK_CUR); + } + + // parse the json object from the file + tok = json_tokener_new(); + do { + rc = read(fd, buffer, len); + if (rc < 0) + break; + obj = json_tokener_parse_ex(tok, buffer, rc); + } while ((jerr = json_tokener_get_error(tok)) == json_tokener_continue); + + if (jerr != json_tokener_success){ + fprintf(stderr, "error in file %s, line: %s\n", filename, buffer); + } + +tok_free: + + json_tokener_free(tok); + +close_fd: + close(fd); + +err: + if (ret) { + fprintf(stderr, "error while reading file: %i", ret); + } + return obj; +} + +// process all ciphersuites one by one from a given host results file +int process_host_results(char *filename) +{ + int fd; + int ret = 0; + int rc; + size_t sz; + size_t alloc_size = 64 * 1024; + const char *str; + struct json_object *root; + struct json_object *ciphers; + struct json_object *current; + struct json_object *certificates; + + struct json_object **known_chains; + known_chains = malloc(sizeof(struct json_object*) * 1); + known_chains[0] = NULL; + + struct lh_table *table; + enum json_type obj_t; + json_bool j_rc; + + root = read_json_from_file(filename); + if (root == NULL) { + ret = 1; + goto err; + } + + obj_t = json_object_get_type(root); + str = json_type_to_name(obj_t); + + j_rc = json_object_object_get_ex(root, "ciphersuite", &ciphers); + if (j_rc == FALSE) { + ret = 1; + goto json_free; + } + + // ok, we've got the ciphersuite part, we can print the json header for + // the host file + printf("{\"host\":\"%s\",\"chains\":[", filename); + + int first_printed=0; + for(int i=0; i < json_object_array_length(ciphers); i++) { + current = json_object_array_get_idx(ciphers, i); + //printf("\t[%i]:\n", i); + j_rc = json_object_object_get_ex(current, "certificates", &certificates); + if (j_rc == FALSE) + continue; + + const char** certs; + certs = calloc(sizeof(const char*), json_object_array_length(certificates) + 1); + int j; + for (j=0; j < json_object_array_length(certificates); j++) { + certs[j] = json_object_get_string(json_object_array_get_idx(certificates, j)); + //printf("\t\t\t%s\n", certs[j]); + } + rc = register_known_chains(&known_chains, certificates); + //printf("\t\t%i\n", rc); + + if (rc == 0 && j > 0) { + if (first_printed != 0) + printf(","); + if (process_chain(certs) != 0) { + fprintf(stderr, "error while processing chains!\n"); + } else { + first_printed = 1; + } + } + + // DEBUG, print whole json "object" object + //json_object_object_foreach(current, key, val) { + // str = json_object_to_json_string(val); + // printf("\t\t%s: %s\n", key, str); + //} + + free(certs); + } + printf("]}"); + +json_free: + json_object_put(root); + +err: + free(known_chains); + return ret; +} + +int main(int argc, char** argv) +{ + int ret; + + DIR *dirp; + struct dirent *direntp; + + char buffer[8192] = {}; + + SSL_load_error_strings(); + SSL_library_init(); + + /* init trust stores with certificate locations */ + trusted_only = SSL_CTX_new(SSLv23_method()); + if (trusted_only == NULL) { + ERR_print_errors_fp(stderr); + return 1; + } + + ret = SSL_CTX_load_verify_locations(trusted_only, NULL, CA_TRUSTED); + if (ret != 1) { + ERR_print_errors_fp(stderr); + return 1; + } + + all_CAs = SSL_CTX_new(SSLv23_method()); + if (all_CAs == NULL) { + ERR_print_errors_fp(stderr); + return 1; + } + + ret = SSL_CTX_load_verify_locations(all_CAs, NULL, CA_ALL); + if (ret != 1) { + ERR_print_errors_fp(stderr); + return 1; + } + + /* traverse the result directory, check all files in turn */ + dirp=opendir("results"); + while((direntp=readdir(dirp)) != NULL) { + if (strcmp(direntp->d_name, ".") == 0) + continue; + if (strcmp(direntp->d_name, "..") == 0) + continue; + + snprintf(buffer, 8191, "results/%s", direntp->d_name); + + ret = process_host_results(buffer); + if (ret == 1) { + fprintf(stderr, "error while processing %s\n", buffer); + } + if (ret == 0) + printf("\n"); + } + closedir(dirp); + + /* clean up */ + SSL_CTX_free(trusted_only); + SSL_CTX_free(all_CAs); + all_CAs = NULL; + trusted_only = NULL; + + return ret; +} diff --git a/top1m/parse_CAs.py b/top1m/parse_CAs.py new file mode 100644 index 0000000..d564c28 --- /dev/null +++ b/top1m/parse_CAs.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Author: Hubert Kario - 2014 + +from __future__ import division + +path = "./results/" +ca_certs_path = "./ca_files" +certs_path = "./certs" + +""" only root CAs, no cached intermediate certs """ +trust_path = "./ca_trusted" + +import json +import sys +from collections import defaultdict +import os +from OpenSSL import crypto + +invocations = defaultdict(int) + +total = 0 +hosts = 0 +chains = defaultdict(int) +chain_len = defaultdict(int) +keysize = defaultdict(int) +keysize_per_chain = defaultdict(int) +root_CA = defaultdict(int) +sig_alg = defaultdict(int) +intermediate_CA = defaultdict(int) +effective_security = defaultdict(int) + +subject_hashes = {} +issuer_hashes = {} + +def get_cert_subject_name(cert): + subject = cert.get_subject() + commonName = None + organization = None + + for elem,val in subject.get_components(): + if elem == "CN" and commonName is None: + commonName = val + if elem == "O" and organization is None: + organization = val + + s_hash = "(" + ("%0.8X" % subject.hash()).lower() + ") " + + if commonName is not None: + return s_hash + commonName + elif organization is not None: + return s_hash + organization + else: + return s_hash + +def get_path_for_hash(cert_hash): + f_name = certs_path + '/' + cert_hash + '.pem' + if not os.path.exists(f_name): + f_name = ca_certs_path + '/' + cert_hash + '.pem' + if not os.path.exists(f_name): + #print "File with hash " + c_hash + " is missing!" + return None + return f_name + +""" convert RSA and DSA key sizes to estimated Level of security """ +def rsa_key_size_to_los(size): + if size < 760: + return 40 + elif size < 1020: + return 64 + elif size < 2040: + return 80 + elif size < 3068: + return 112 + elif size < 4094: + return 128 + elif size < 7660: + return 152 + elif size < 15300: + return 192 + else: + return 256 + +def sig_alg_to_los(name): + if 'MD5' in name.upper(): + return 64 + elif 'SHA1' in name.upper(): + return 80 + elif 'SHA224' in name.upper(): + return 112 + elif 'SHA256' in name.upper(): + return 128 + elif 'SHA384' in name.upper(): + return 192 + elif 'SHA512' in name.upper(): + return 256 + else: + raise UnknownSigAlgError + +def collect_key_sizes(file_names): + + tmp_keysize = {} + + """ don't collect signature alg for the self signed root """ + with open(file_names[-1]) as cert_file: + cert_pem = cert_file.read() + + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + pubkey = cert.get_pubkey() + if pubkey.type() == crypto.TYPE_RSA: + keysize['RSA ' + str(pubkey.bits())] += 1 + tmp_keysize['RSA ' + str(pubkey.bits())] = 1 + security_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == crypto.TYPE_DSA: + keysize['DSA ' + str(pubkey.bits())] += 1 + tmp_keysize['DSA ' + str(pubkey.bits())] = 1 + security_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == 408: + keysize['ECDSA ' + str(pubkey.bits())] += 1 + tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1 + security_level = pubkey.bits()/2 + else: + keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1 + security_level = 0 + + root_CA[get_cert_subject_name(cert)] += 1 + + """ exclude the self signed root and server cert from stats """ + for f_name in file_names[1:-1]: + with open(f_name) as cert_file: + cert_pem = cert_file.read() + + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + pubkey = cert.get_pubkey() + if pubkey.type() == crypto.TYPE_RSA: + keysize['RSA ' + str(pubkey.bits())] += 1 + tmp_keysize['RSA ' + str(pubkey.bits())] = 1 + c_key_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == crypto.TYPE_DSA: + keysize['DSA ' + str(pubkey.bits())] += 1 + tmp_keysize['DSA ' + str(pubkey.bits())] = 1 + c_key_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == 408: + keysize['ECDSA ' + str(pubkey.bits())] += 1 + tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1 + c_key_level = pubkey.bits() / 2 + else: + keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1 + c_key_level = 0 + + if security_level > c_key_level: + security_level = c_key_level + + sig_alg[cert.get_signature_algorithm()] += 1 + c_sig_level = sig_alg_to_los(cert.get_signature_algorithm()) + if security_level > c_sig_level: + security_level = c_sig_level + + intermediate_CA[get_cert_subject_name(cert)] += 1 + + for key_s in tmp_keysize: + keysize_per_chain[key_s] += 1 + + # XXX doesn't handle the situation in which the CA uses its certificate + # for a web server properly + with open(file_names[0]) as cert_file: + cert_pem = cert_file.read() + + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + pubkey = cert.get_pubkey() + if pubkey.type() == crypto.TYPE_RSA: + c_key_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == crypto.TYPE_DSA: + c_key_level = rsa_key_size_to_los(pubkey.bits()) + elif pubkey.type() == 408: + c_key_level = pubkey.bits() / 2 + else: + c_key_level = 0 + + if security_level > c_key_level: + security_level = c_key_level + + c_sig_level = sig_alg_to_los(cert.get_signature_algorithm()) + if security_level > c_sig_level: + security_level = c_sig_level + + effective_security[security_level] += 1 + + +with open("parsed") as res_file: + for line in res_file: + try: + res = json.loads(line) + except ValueError as e: + print "can't process line: " + line + continue + + f=res + + try: + server_chain_trusted = False + server_chain_complete = False + server_chains = [] + valid = False + + """ Keep certificates in memory for a given file """ + known_certs = {} + + if not "chains" in f: + continue + + results = f["chains"] + + """ discard hosts with empty results """ + if len(results) < 1: + continue + + """ loop over list of ciphers """ + for entry in results: + + """ skip invalid results """ + if not 'chain' in entry: + continue + + valid = True + + if entry['chain'] == "untrusted": + continue + + if entry['chain'] == "complete": + server_chain_complete = True + server_chain_trusted = True + + if entry['chain'] == "incomplete": + server_chain_trusted = True + + server_chains += [entry['certificates']] + + if server_chain_trusted: + if server_chain_complete: + chains["complete"] += 1 + print "complete: " + f['host'] + else: + chains["incomplete"] += 1 + print "incomplete: " + f['host'] + else: + chains["untrusted"] += 1 + print "untrusted: " + f['host'] + + if valid: + hosts += 1 + + for chain in server_chains: + f_names = [] + for hash in chain: + path = get_path_for_hash(hash) + f_names += [path] + + collect_key_sizes(f_names) + chain_len[str(len(chain))] += 1 + if len(chain) == 1: + sys.stderr.write("file with chain 1 long: " + line) + total += 1 + except TypeError as e: + + sys.stderr.write("can't process: " + line) + continue + +""" Display stats """ +#print "openssl invocations: " + str(invocations["openssl"]) + +print "Statistics from " + str(total) + " chains provided by " + str(hosts) + " hosts" + +print("\nServer provided chains Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(chains): + percent = round(chains[stat] / hosts * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(chains[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nTrusted chain statistics") +print("========================") + + +print("\nChain length Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(chain_len): + percent = round(chain_len[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(chain_len[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nCA key size in chains Count") +print("-------------------------+---------") +for stat in sorted(keysize): + sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + "\n") + +print("\nChains with CA key Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(keysize_per_chain): + percent = round(keysize_per_chain[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(keysize_per_chain[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nSignature algorithm (ex. root) Count") +print("------------------------------+---------") +for stat in sorted(sig_alg): + sys.stdout.write(stat.ljust(30) + " " + str(sig_alg[stat]).ljust(10) + "\n") + +print("\nEff. host cert chain LoS Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(effective_security): + percent = round(effective_security[stat] / total * 100, 4) + sys.stdout.write(str(stat).ljust(25) + " " + str(effective_security[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nRoot CAs Count Percent") +print("---------------------------------------------+---------+-------") +for stat in sorted(root_CA): + percent = round(root_CA[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(45)[0:45] + " " + str(root_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nIntermediate CA Count Percent") +print("---------------------------------------------+---------+-------") +for stat in sorted(intermediate_CA): + percent = round(intermediate_CA[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(45)[0:45] + " " + str(intermediate_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n") diff --git a/top1m/parse_results.py b/top1m/parse_results.py index 40d85f6..0ac7c4f 100644 --- a/top1m/parse_results.py +++ b/top1m/parse_results.py @@ -14,10 +14,47 @@ import json import sys from collections import defaultdict import os +import re + +def natural_sort(l): + convert = lambda text: int(text) if text.isdigit() else text.lower() + alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] + return sorted(l, key = alphanum_key) + +""" list of ciphers offerred by Firefox 29 by default """ +firefox_ciphers=[ + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES256-SHA', + 'ECDHE-ECDSA-AES128-SHA', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-RSA-AES256-SHA', + 'ECDHE-RSA-DES-CBC3-SHA', + 'ECDHE-ECDSA-RC4-SHA', + 'ECDHE-RSA-RC4-SHA', + 'DHE-RSA-AES128-SHA', + 'DHE-DSS-AES128-SHA', + 'DHE-RSA-CAMELLIA128-SHA', + 'DHE-RSA-AES256-SHA', + 'DHE-DSS-AES256-SHA', + 'DHE-RSA-CAMELLIA256-SHA', + 'EDH-RSA-DES-CBC3-SHA', + 'AES128-SHA', + 'CAMELLIA128-SHA', + 'AES256-SHA', + 'CAMELLIA256-SHA', + 'DES-CBC3-SHA', + 'RC4-SHA', + 'RC4-MD5'] report_untrused=False cipherstats = defaultdict(int) +FF_RC4_Only_cipherstats = defaultdict(int) +FF_RC4_preferred_cipherstats = defaultdict(int) +FF_incompatible_cipherstats = defaultdict(int) +FF_selected_cipherstats = defaultdict(int) +cipherordering = defaultdict(int) pfsstats = defaultdict(int) protocolstats = defaultdict(int) handshakestats = defaultdict(int) @@ -38,6 +75,7 @@ for r,d,flist in os.walk(path): tempdsakeystats = {} tempsigstats = {} tempticketstats = {} + """ supported ciphers by the server under scan """ tempcipherstats = {} ciphertypes = 0 AESGCM = False @@ -46,6 +84,13 @@ for r,d,flist in os.walk(path): DES3 = False CAMELLIA = False RC4 = False + """ the following depends on FF_compat, so by default it can be True """ + RC4_Only_FF = True + FF_compat = False + temp_FF_incompat = {} + list_of_ciphers = [] + FF_RC4_Pref = None + FF_selected = None ADH = False DHE = False AECDH = False @@ -85,11 +130,25 @@ for r,d,flist in os.walk(path): if 'False' in entry['trusted'] and report_untrused == False: continue + list_of_ciphers.append(entry['cipher']) + + # check if the advertised ciphers are not effectively RC4 Only + # for firefox or incompatible with firefox + if entry['cipher'] in firefox_ciphers: + # if this is first cipher and we already are getting RC4 + # then it means that RC4 is preferred + FF_compat = True + if not 'RC4' in entry['cipher']: + RC4_Only_FF = False + else: + temp_FF_incompat[entry['cipher']] = 1 + """ store the ciphers supported """ if 'ADH' in entry['cipher'] or 'AECDH' in entry['cipher']: ciphertypes += 1 name = "z:" + entry['cipher'] tempcipherstats[name] = 1 + tempcipherstats['Insecure'] = 1 elif 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']: if not AESGCM: AESGCM = True @@ -114,10 +173,15 @@ for r,d,flist in os.walk(path): if not CHACHA20: ciphertypes += 1 CHACHA20 = True + elif 'IDEA' in entry['cipher'] or 'SEED' in entry['cipher']: + ciphertypes += 1 + name = "y:" + entry['cipher'] + tempcipherstats[name] = 1 else: ciphertypes += 1 name = "z:" + entry['cipher'] tempcipherstats[name] = 1 + tempcipherstats['Insecure'] = 1 """ store key handshake methods """ if 'ECDHE' in entry['cipher']: @@ -138,7 +202,7 @@ for r,d,flist in os.walk(path): RSA = True """ save the key size """ - if 'ECDSA' in entry['cipher']: + if 'ECDSA' in entry['cipher'] or 'ECDH-RSA' in entry['cipher']: ECDSA = True tempecckeystats[entry['pubkey'][0]] = 1 elif 'DSS' in entry['cipher']: @@ -206,6 +270,32 @@ for r,d,flist in os.walk(path): if dualstack: dsarsastack += 1 + """ save cipher ordering """ + if 'serverside' in results: + if results['serverside'] == "False": + cipherordering['Client side'] += 1 + else: + cipherordering['Server side'] += 1 + else: + cipherordering['Unknown'] += 1 + + """ simulate handshake with Firefox """ + if FF_compat: + if 'serverside' in results and results['serverside'] == "False": + for cipher in firefox_ciphers: + if cipher in list_of_ciphers: + FF_selected = cipher + if 'RC4' in cipher: + FF_RC4_Pref = True + break + else: + for cipher in list_of_ciphers: + if cipher in firefox_ciphers: + FF_selected = cipher + if 'RC4' in cipher: + FF_RC4_Pref = True + break + for s in tempsigstats: sigalg[s] += 1 @@ -256,6 +346,27 @@ for r,d,flist in os.walk(path): cipherstats['RC4 forced in TLS1.1+'] += 1 cipherstats['RC4 Preferred'] += 1 + if FF_compat: + if 'ECDHE' in FF_selected: + FF_selected_cipherstats['x:ECDHE'] += 1 + elif 'DHE' in FF_selected or 'EDH' in FF_selected: + FF_selected_cipherstats['x:DHE'] += 1 + else: + FF_selected_cipherstats['x:kRSA'] += 1 + FF_selected_cipherstats[FF_selected] += 1 + if RC4_Only_FF and ciphertypes != 1: + cipherstats['x:FF 29 RC4 Only'] += 1 + for cipher in temp_FF_incompat: + FF_RC4_Only_cipherstats[cipher] += 1 + if FF_RC4_Pref and not 'RC4' in results['ciphersuite'][0]['cipher']: + cipherstats['x:FF 29 RC4 Preferred'] += 1 + for cipher in temp_FF_incompat: + FF_RC4_preferred_cipherstats[cipher] += 1 + else: + cipherstats['x:FF 29 incompatible'] += 1 + for cipher in temp_FF_incompat: + FF_incompatible_cipherstats[cipher] += 1 + for cipher in tempcipherstats: cipherstats[cipher] += 1 @@ -309,6 +420,13 @@ for r,d,flist in os.walk(path): #if total % 1999 == 0: # break +""" The 'x:FF 29 RC4 Preferred' counts only sites that effectively prefer + RC4 when using FF, to make reporting more readable, sum it with sites + that do that for all ciphers""" + +if "x:FF 29 RC4 Preferred" in cipherstats and "RC4 Preferred" in cipherstats: + cipherstats['x:FF 29 RC4 Preferred'] += cipherstats['RC4 Preferred'] + print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total) if report_untrused == False: print("Stats only from connections that did provide valid certificates") @@ -321,6 +439,36 @@ for stat in sorted(cipherstats): percent = round(cipherstats[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") +print("\nCipher ordering Count Percent") +print("-------------------------+---------+-------") +for stat in sorted(cipherordering): + percent = round(cipherordering[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(25) + " " + str(cipherordering[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nFF 29 selected ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_selected_cipherstats): + percent = round(FF_selected_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_selected_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nFF 29 RC4 Only other ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_RC4_Only_cipherstats): + percent = round(FF_RC4_Only_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_Only_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nFF 29 RC4 pref other ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_RC4_preferred_cipherstats): + percent = round(FF_RC4_preferred_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_preferred_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + +print("\nFF 29 incompatible ciphers Count Percent") +print("-----------------------------+---------+------") +for stat in sorted(FF_incompatible_cipherstats): + percent = round(FF_incompatible_cipherstats[stat] / total * 100, 4) + sys.stdout.write(stat.ljust(30) + " " + str(FF_incompatible_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n") + print("\nSupported Handshakes Count Percent") print("-------------------------+---------+-------") for stat in sorted(handshakestats): @@ -340,7 +488,7 @@ for stat in sorted(pfsstats): print("\nTLS session ticket hint Count Percent ") print("-------------------------+---------+--------") -for stat in sorted(tickethint): +for stat in natural_sort(tickethint): percent = round(tickethint[stat] / total * 100, 4) sys.stdout.write(stat.ljust(25) + " " + str(tickethint[stat]).ljust(10) + str(percent).ljust(9) + "\n") diff --git a/top1m/process-certificate-statistics.sh b/top1m/process-certificate-statistics.sh new file mode 100755 index 0000000..b9256d8 --- /dev/null +++ b/top1m/process-certificate-statistics.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +if [ ! -d ./ca_files ]; then + echo "Directory with collected CA certificates missing!" >&2 + exit 1 +fi + +if [ ! -d ./ca_trusted ]; then + echo "Directory with just trust anchors missing!" >&2 + exit 1 +fi + +if [ ! -d ./certs ]; then + echo "Directory with certificates missing!" >&2 + exit 1 +fi + +if ! ls -f ./ca_files/????????.? > /dev/null; then + echo "CA certificates directory not hashed properly (use c_rehash)" >&2 + exit 1 +fi + +if ! ls -f ./ca_trusted/????????.? > /dev/null; then + echo "Directory with trust anchors not hashed properly (use c_rehash)" >&2 + exit 1 +fi + +if [ ! -d ./results ]; then + echo "Directory with scan results missing!" >&2 + exit 1 +fi + +if [ ! -x ./parse_CAs ]; then + echo "Compiling parse_CAs script" + gcc -o parse_CAs parse_CAs.c -lssl -lcrypto -ljson-c --std=gnu99 + if [ $? -ne 0 ]; then + echo "Compilation failed, aborting" >&2 + exit 1 + fi +fi + +echo "Verifying certificate chains from results files" +./parse_CAs > parsed +echo "Calculating statistics for verified certificate chains" +python parse_CAs.py > trust_scan +echo "Done!" +echo "Results are in \"trust_scan\" file" diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh index b2cb0ef..6021a3e 100755 --- a/top1m/testtop1m.sh +++ b/top1m/testtop1m.sh @@ -2,20 +2,44 @@ parallel=10 max_bg=50 absolute_max_bg=100 -max_load=50 +max_load_avg=50 if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then echo "max user processes too low, use ulimit -u to increase" exit 1 fi [ ! -e "results" ] && mkdir results +[ ! -e "certs" ] && mkdir certs +if [ ! -e "ca_files" ]; then + mkdir ca_files + pushd ca_files >/dev/null + awk ' + split_after == 1 {n++;split_after=0} + /-----END CERTIFICATE-----/ {split_after=1} + {print > "cert" n ".pem"}' < "/etc/pki/tls/certs/ca-bundle.crt" + for i in *; do + h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null) + for num in `seq 0 100`; do + if [[ $h.$num -ef $i ]]; then + # file already linked, ignore + break + fi + if [[ ! -e $h.$num ]]; then + # file doesn't exist, create a link + ln -s "$i" "$h.$num" + break + fi + done + done + popd >/dev/null +fi function wait_for_jobs() { local no_jobs no_jobs=$(jobs | wc -l) - while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do - if awk -v maxload=$max_load '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then + while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load_avg '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do + if awk -v maxload=$max_load_avg '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then return fi sleep 1 @@ -32,7 +56,7 @@ function scan_host() { if [ $? -gt 0 ]; then return fi - ../cipherscan -json -servername $1 $2:443 > results/$1@$2 + ../cipherscan --capath ca_files --savecrt certs -json -servername $1 $2:443 > results/$1@$2 } function scan_host_no_sni() { @@ -44,10 +68,12 @@ function scan_host_no_sni() { if [ $? -gt 0 ]; then return fi - ../cipherscan -json $1:443 > results/$1 + ../cipherscan --capath ca_files --savecrt certs -json $1:443 > results/$1 } function scan_hostname() { + # check if the hostname isn't an IP address (since we can't put IP + # addresses to SNI extension) if [[ ! -z $(awk -F. '$1>=0 && $1<=255 && $2>=0 && $2<=255 && $3>=0 && $3<=255 && $4>=0 && $4<=255 && NF==4' <<<"$1") ]]; then scan_host_no_sni $1