Merge 77f326522e into ded65c40df

cipherscan

@@ -22,6 +22,8 @@ fi
 if [ ! -e "$CACERTS" ]; then
     echo "Warning: CA Certificates not found at $CACERTS, export CACERTS variable with location of your trust anchors" 1>&2
 fi
+# RSA ciphers are put at the end to force Google servers to accept ECDSA ciphers
+# (probably a result of a workaround for the bug in Apple implementation of ECDSA)
 CIPHERSUITE="ALL:COMPLEMENTOFALL:+aRSA"
 DEBUG=0
 VERBOSE=0
@@ -29,10 +31,19 @@ DELAY=0
 ALLCIPHERS=0
 OUTPUTFORMAT="terminal"
 TIMEOUT=10
-
+# place where to put the found intermediate CA certificates and where
+# trust anchors are stored
+CAPATH=""
+SAVECRT=""
+unset ok_protocols
+declare -A ok_protocols
+unset known_certs
+declare -A known_certs
+unset cert_checksums
+declare -A cert_checksums
 
 usage() {
-    echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [-d|--delay seconds] [-D|--debug] [-j|--json] [-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
+    echo -e "usage: $0 [-a|--allciphers] [-b|--benchmark] [--capath directory] [-d|--delay seconds] [-D|--debug] [-j|--json] [--savecrt directory] [-v|--verbose] [-o|--openssl file] [openssl s_client args] <target:port>
     usage: $0 -h|--help
 
 $0 attempts to connect to a target site using all the ciphersuites it knows.
@@ -46,11 +57,13 @@ Use one of the options below:
 
 -a | --allciphers   Test all known ciphers individually at the end.
 -b | --benchmark    Activate benchmark mode.
+--capath            use CAs from directory, save intermediate certificates there
 -d | --delay        Pause for n seconds between connections
 -D | --debug        Output ALL the information.
 -h | --help         Shows this help text.
 -j | --json         Output results in JSON format.
 -o | --openssl      path/to/your/openssl binary you want to use.
+--savecrt           path where to save untrusted and leaf certificates
 -v | --verbose      Increase verbosity.
 
 The rest of the arguments will be interpreted as openssl s_client argument.
@@ -74,17 +87,47 @@ debug(){
     fi
 }
 
+c_hash() {
+    local h=$(${OPENSSLBIN} x509 -hash -noout -in "$1/$2" 2>/dev/null)
+    for num in $(seq 0 100); do
+        if [[ $1/${h}.${num} -ef $2 ]]; then
+            # file already linked, ignore
+            break
+        fi
+        if [[ ! -e $1/${h}.${num} ]]; then
+            # file doesn't exist, create a link
+            pushd "$1" > /dev/null
+            ln -s "$2" "${h}.${num}"
+            popd > /dev/null
+            break
+        fi
+    done
+}
+
 # Connect to a target host with the selected ciphersuite
 test_cipher_on_target() {
     local sslcommand=$@
     cipher=""
+    local cmnd=""
     protocols=""
     pfs=""
     previous_cipher=""
+    certificates=""
     for tls_version in "-ssl2" "-ssl3" "-tls1" "-tls1_1" "-tls1_2"
     do
-        debug echo \"Q\" \| $sslcommand $tls_version
+        if [[ ${ok_protocols[$tls_version]} -eq 1 ]]; then
-        local tmp=$(echo "Q" | $sslcommand $tls_version 1>/dev/stdout 2>/dev/null)
+            continue
+        fi
+        # sslv2 client hello doesn't support SNI extension
+        # in SSLv3 mode OpenSSL just ignores the setting so it's ok
+        # -status exception is ignored in SSLv2, go figure
+        if [ "$tls_version" == "-ssl2" ]; then
+            cmnd=$(sed 's/-servername\ [^ ]*//'<<<$sslcommand)
+        else
+            cmnd=$sslcommand
+        fi
+        debug echo \"Q\" \| $cmnd $tls_version
+        local tmp=$(echo "Q" | $cmnd $tls_version 1>/dev/stdout 2>/dev/null)
         if grep 'OCSP Response Data' <<<"$tmp" >/dev/null; then
             current_ocspstaple="True"
         else
@@ -92,18 +135,23 @@ test_cipher_on_target() {
         fi
         # filter out the OCSP server certificate
         tmp=$(awk 'BEGIN { pr="yes" } /^======================================/ { if ( pr=="yes" ) pr="no"; else pr="yes" } { if ( pr == "yes" ) print }' <<<"$tmp")
-        current_cipher=$(grep "New, " <<<"$tmp"|awk '{print $5}')
+
-        current_pfs=$(grep 'Server Temp Key' <<<"$tmp"|awk '{print $4$5$6$7}')
+        # session metadata
-        current_protocol=$(egrep "^\s+Protocol\s+:" <<<"$tmp"|awk '{print $3}')
+        current_cipher=$(awk '/New, / {print $5; exit}' <<<"$tmp")
-        current_pubkey=$(grep 'Server public key is ' <<<"$tmp"|awk '{print $5}')
+        current_pfs=$(awk '/Server Temp Key/ {print $4$5$6$7; exit}' <<<"$tmp")
-        if [ -z $current_pubkey ]; then
+        current_protocol=$(awk '/^\s+Protocol\s+:/ {print $3; exit}' <<<"$tmp")
-            current_pubkey=0
+        current_tickethint=$(awk '/ticket lifetime hint/ {print $6; exit}' <<<"$tmp")
-        fi
-        current_tickethint=$(grep 'ticket lifetime hint' <<<"$tmp"|awk '{print $6 }')
         if [ -z $current_tickethint ]; then
             current_tickethint=None
         fi
-        current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|grep Signature\ Algorithm | head -n 1 | awk '{print $3}') || current_sigalg="None"
+
+        # certificate metadata
+        current_pubkey=$(awk '/Server public key is / {print $5;exit}' <<<"$tmp")
+        if [ -z $current_pubkey ]; then
+            current_pubkey=0
+        fi
+        current_sigalg=$(${OPENSSLBIN} x509 -noout -text 2>/dev/null <<<"$tmp"|\
+            awk '/Signature Algorithm/ {print $3; exit}') || current_sigalg="None"
         grep 'Verify return code: 0 ' <<<"$tmp" >/dev/null
         if [ $? -eq 0 ]; then
             current_trusted="True"
@@ -113,7 +161,88 @@ test_cipher_on_target() {
         if [ -z $current_sigalg ]; then
             current_sigalg=None
         fi
+
+        # collect certificate data
+        current_certificates=""
+        local certificate_count=$(grep --count -- '-----END CERTIFICATE-----'\
+            <<<"$tmp")
+        debug "server presented $certificate_count certificates"
+        local i
+        for ((i=0; i<$certificate_count; i=i+1 )); do
+
+            # extract i'th certificate
+            local cert=$(awk -v i=$i 'BEGIN { output=0;n=0 }
+            /-----BEGIN CERTIFICATE-----/ { output=1 }
+            output==1 { if (n==i) print }
+            /-----END CERTIFICATE-----/ { output=0; n++ }' <<<"$tmp")
+            # put the output to an array instead running awk '{print $1}'
+            local cksum=($(cksum <<<"$cert"))
+            # compare the values not just checksums so that eventual collision
+            # doesn't mess up results
+            if [[ ${known_certs[$cksum]} == $cert ]]; then
+                if [ -n "${current_certificates}" ]; then
+                    current_certificates+=","
+                fi
+                current_certificates+="\"${cert_checksums[$cksum]}\""
+                continue
+            fi
+
+            # compute sha256 fingerprint of the certificate
+            local sha256sum=$(${OPENSSLBIN} x509 -outform DER\
+                <<<"$cert" 2>/dev/null |\
+                ${OPENSSLBIN} dgst -sha256 -r 2>/dev/null| awk '{print $1}')
+
+            # check if it is a CA certificate
+            local isCA="False"
+            if ${OPENSSLBIN} x509 -noout -text <<<"$cert" 2>/dev/null |\
+                grep 'CA:TRUE' >/dev/null; then
+                isCA="True"
+            fi
+
+            # build trust source for certificate verification
+            local trust_source=()
+            if [[ -n $CAPATH ]]; then
+                trust_source=("-CApath" "$CAPATH")
+            elif [[ -e $CACERTS ]]; then
+                trust_source=("-CAfile" "$CACERTS")
+            fi
+
+            # check if the certificate is actually trusted (server may present
+            # unrelated certificates that are not trusted (including self
+            # signed ones)
+            local saved="False"
+            if ${OPENSSLBIN} verify "${trust_source[@]}" \
+                -untrusted <(echo "$tmp") <(echo "$cert") 2>/dev/null | \
+                grep ': OK$' >/dev/null; then
+
+                # if the certificate is an intermediate CA it may be useful
+                # for connecting to servers that are misconfigured so save it
+                if [[ -n $CAPATH ]] && [[ $isCA == "True" ]]; then
+                    if [[ ! -e "$CAPATH/${sha256sum}.pem" ]]; then
+                        echo "$cert" > "$CAPATH/${sha256sum}.pem"
+                        c_hash "$CAPATH" "${sha256sum}.pem"
+                    fi
+                    saved="True"
+                fi
+            fi
+            if [[ -n $SAVECRT ]] && [[ $saved == "False" ]]; then
+                if [[ ! -e $SAVECRT/${sha256sum}.pem ]]; then
+                    echo "$cert" > "$SAVECRT/${sha256sum}.pem"
+                fi
+            fi
+            # save the sha sum for reporting
+            if [ -n "${current_certificates}" ]; then
+                current_certificates+=","
+            fi
+            current_certificates+="\"${sha256sum}\""
+            known_certs[$cksum]="$cert"
+            cert_checksums[$cksum]="$sha256sum"
+        done
+        debug "current_certificates: $current_certificates"
+
+        # parsing finished, report result
         if [[ -z "$current_protocol" || "$current_cipher" == '(NONE)' ]]; then
+            ok_protocols["$tls_version"]=1
             # connection failed, try again with next TLS version
             continue
         else
@@ -138,6 +267,7 @@ test_cipher_on_target() {
         trusted=$current_trusted
         tickethint=$current_tickethint
         ocspstaple=$current_ocspstaple
+        certificates="$current_certificates"
         # grab the cipher and PFS key size
     done
     # if cipher is empty, that means none of the TLS version worked with
@@ -182,24 +312,28 @@ bench_cipher() {
     cipherbenchms="$((t/1000/$BENCHMARKITER))"
 }
 
-
 # Connect to the target and retrieve the chosen cipher
 # recursively until the connection fails
 get_cipher_pref() {
     [ "$OUTPUTFORMAT" == "terminal" ] && [ $DEBUG -lt 1 ] && echo -n '.'
     local ciphersuite="$1"
-    if [ -e $CACERTS ]; then
+
-        local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -CAfile $CACERTS -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+    local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client"
-    else
+    if [ -n "$CAPATH" ]; then
-        local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+        sslcommand+=" -CApath $CAPATH -showcerts"
+    elif [ -e $CACERTS ]; then
+        sslcommand+=" -CAfile $CACERTS"
     fi
+    sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+
     verbose "Connecting to '$TARGET' with ciphersuite '$ciphersuite'"
     test_cipher_on_target "$sslcommand"
     local success=$?
     # If the connection succeeded with the current cipher, benchmark and store
     if [ $success -eq 0 ]; then
         cipherspref=("${cipherspref[@]}" "$result")
-        pciph=$(echo $result|awk '{print $1}')
+        ciphercertificates=("${ciphercertificates[@]}" "$certificates")
+        pciph=($(echo $result))
         get_cipher_pref "!$pciph:$ciphersuite"
         return 0
     fi
@@ -217,7 +351,7 @@ display_results_in_terminal() {
     local ocspstaple
     local different=False
     for cipher in "${cipherspref[@]}"; do
-        pciph=$(echo $cipher|awk '{print $1}')
+        pciph=($(echo $cipher))
         if [ $DOBENCHMARK -eq 1 ]; then
             bench_cipher "$pciph"
             r="$ctr $cipher $cipherbenchms"
@@ -250,13 +384,13 @@ display_results_in_terminal() {
 
     if [ $DOBENCHMARK -eq 1 ]; then
         if [ $different == "True" ]; then
-            header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint pfs_keysize avg_handshake_microsec"
+            header="prio ciphersuite protocols pubkey_size signature_algoritm trusted ticket_hint ocsp_staple pfs_keysize avg_handshake_microsec"
         else
             header="prio ciphersuite protocols pfs_keysize avg_handshake_microsec"
         fi
     else
         if [ $different == "True" ]; then
-            header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint pfs_keysize"
+            header="prio ciphersuite protocols pubkey_size signature_algorithm trusted ticket_hint ocsp_staple pfs_keysize"
         else
             header="prio ciphersuite protocols pfs_keysize"
         fi
@@ -268,10 +402,14 @@ display_results_in_terminal() {
             ctr=$((ctr+1))
         fi
         if [ $different == "True" ]; then
-            echo $result|grep -v '(NONE)'
+            if [[ $(awk '{print $3}' <<< $result) == "SSLv3,TLSv1,TLSv1.1,TLSv1.2" ]]; then
+                awk '!/(NONE)/{print $1 " " $2 " " "SSLv3-TLSv1.2" " " $4 " " $5 " " $6 " " $7 " " $8 " " $9}' <<<"$result"
+            else
+                echo $result|grep -v '(NONE)'
+            fi
         else
             # prints priority, ciphersuite, protocols and pfs_keysize
-            echo $result|grep -v '(NONE)'|awk '{print $1 " " $2 " " $3 " " $9}'
+            awk '!/(NONE)/{print $1 " " $2 " " $3 " " $9}' <<<"$result"
         fi
     done|column -t
     echo
@@ -288,13 +426,18 @@ display_results_in_terminal() {
     else
         echo "OCSP stapling: not supported"
     fi
+    if [[ $serverside == "True" ]]; then
+        echo "Server side cipher ordering"
+    else
+        echo "Client side cipher ordering"
+    fi
 }
 
 
 display_results_in_json() {
     # Display the results in json
     ctr=0
-    echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"ciphersuite\": ["
+    echo -n "{\"target\":\"$TARGET\",\"date\":\"$(date -R)\",\"serverside\":\"${serverside}\",\"ciphersuite\": ["
     for cipher in "${cipherspref[@]}"; do
         [ $ctr -gt 0 ] && echo -n ','
         echo -n "{\"cipher\":\"$(echo $cipher|awk '{print $1}')\","
@@ -302,6 +445,9 @@ display_results_in_json() {
         echo -n "\"pubkey\":[\"$(echo $cipher|awk '{print $3}'|sed 's/,/","/g')\"],"
         echo -n "\"sigalg\":[\"$(echo $cipher|awk '{print $4}'|sed 's/,/","/g')\"],"
         echo -n "\"trusted\":\"$(echo $cipher|awk '{print $5}'|sed 's/,/","/g')\","
+        if [[ -n $CAPATH ]]; then
+            echo -n "\"certificates\":[${ciphercertificates[$ctr]}],"
+        fi
         echo -n "\"ticket_hint\":\"$(echo $cipher|awk '{print $6}')\","
         echo -n "\"ocsp_stapling\":\"$(echo $cipher|awk '{print $7}')\","
         pfs=$(echo $cipher|awk '{print $8}')
@@ -312,6 +458,60 @@ display_results_in_json() {
     echo ']}'
 }
 
+test_serverside_ordering() {
+
+    local ciphersuite=""
+    local prefered=""
+    # server supports only one cipher or no ciphers, so it effectively uses server side ordering...
+    if [[ ${#cipherspref[@]} -lt 2 ]]; then
+        serverside="True"
+        return 0
+    # server supports just two ciphers, so rotate them, that should be enough
+    elif [[ ${#cipherspref[@]} -eq 2 ]]; then
+
+        local cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
+        prefered="$cipher"
+        ciphersuite=$cipher
+
+        cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
+        ciphersuite+=":$cipher"
+
+    # server supports 3 or more ciphers, rotate all three. This is necessary because google does
+    # select first client provided cipher, if it is either CDHE-RSA-AES128-GCM-SHA256 or
+    # ECDHE-RSA-CHACHA20-POLY1305
+    else
+        local cipher=$(awk '{print $1}' <<< ${cipherspref[2]})
+        prefered="$cipher"
+        ciphersuite="$cipher"
+
+        cipher=$(awk '{print $1}' <<< ${cipherspref[1]})
+        ciphersuite+=":$cipher"
+
+        cipher=$(awk '{print $1}' <<< ${cipherspref[0]})
+        ciphersuite+=":$cipher"
+    fi
+
+    local sslcommand="timeout $TIMEOUT $OPENSSLBIN s_client"
+    if [ -n "$CAPATH" ]; then
+        sslcommand+=" -CApath $CAPATH -showcerts"
+    elif [ -e "$CACERTS" ]; then
+        sslcommand+=" -CAfile $CACERTS"
+    fi
+    sslcommand+=" -status $SCLIENTARGS -connect $TARGET -cipher $ciphersuite"
+
+    test_cipher_on_target "$sslcommand"
+    if [ $? -ne 0 ]; then
+        serverside="True"
+    else
+        local selected=$(awk '{print $1}' <<< $result)
+        if [[ $selected == $prefered ]]; then
+            serverside="False"
+        else
+            serverside="True"
+        fi
+    fi
+}
+
 # UNKNOWNOPTIONS=""
 while :
 do
@@ -349,6 +549,14 @@ do
             DELAY=$2
             shift 2
             ;;
+        --capath)
+            CAPATH="$2"
+            shift 2
+            ;;
+        --savecrt)
+            SAVECRT="$2"
+            shift 2
+            ;;
         --) # End of all options
             shift
             break
@@ -393,10 +601,15 @@ debug "sclientargs: $SCLIENTARGS"
 
 
 cipherspref=();
+ciphercertificates=()
 results=()
 
 # Call to the recursive loop that retrieves the cipher preferences
 get_cipher_pref $CIPHERSUITE
+unset ok_protocols
+declare -A ok_protocols
+
+test_serverside_ordering
 
 if [ "$OUTPUTFORMAT" == "json" ]; then
     display_results_in_json

top1m/parse_CAs.c

@@ -0,0 +1,509 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ * Author: Hubert Kario - 2014
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <openssl/x509.h>
+#include <openssl/ssl.h>
+#include <openssl/x509v3.h>
+#include <openssl/stack.h>
+#include <openssl/err.h>
+#include <json-c/json.h>
+
+static char* CA_TRUSTED = "./ca_trusted";
+static char* CA_ALL = "./ca_files";
+static char* CERTS_DIR = "./certs";
+
+/* SSL context that knows only about trust anchors */
+SSL_CTX *trusted_only;
+/* SSL context that also has access to other CA certs */
+SSL_CTX *all_CAs;
+
+// load certificate from file to a OpenSSL object
+X509 *load_cert(char *filename)
+{
+    BIO* f;
+    X509 *ret;
+
+    f = BIO_new(BIO_s_file());
+    BIO_read_filename(f, filename);
+
+    ret = PEM_read_bio_X509_AUX(f, NULL, 0, NULL);
+    if (ret == NULL)
+        fprintf(stderr, "Unable to load file %s as X509 certificate\n", filename);
+
+    BIO_free_all(f);
+
+    return ret;
+}
+
+// convert sha256 to a file name, if the file exists
+// search in "all CAs" dir and "leaf certs" directories
+char *hash_to_filename(const char *hash)
+{
+    char *tmp_f_name;
+    size_t n;
+
+    n = strlen(hash) + 30;
+
+    // TODO error checking
+    tmp_f_name = malloc(n);
+
+    /* first check if the file is in directory with regular certs */
+    // TODO error checking
+    snprintf(tmp_f_name, n, "%s/%s.pem", CERTS_DIR, hash);
+    if (access(tmp_f_name, F_OK) != -1) {
+        return tmp_f_name;
+    }
+
+    snprintf(tmp_f_name, n, "%s/%s.pem", CA_ALL, hash);
+    if (access(tmp_f_name, F_OK) != -1) {
+        return tmp_f_name;
+    }
+
+    // file not found
+    free(tmp_f_name);
+    return NULL;
+}
+
+// take certificate hashes, check their validity and output json that
+// will indicate which certificate were used for verification, whatever
+// the chain was trusted and if all certificates needed for verification
+// (with the exception of root CA) were present in hashes
+int process_chain(const char **cert_hashes)
+{
+    int ret;
+    int rc; // return code from function
+    char *f_name;
+
+    X509 *cert;
+    X509 *x509;
+
+    X509_STORE *store;
+
+    X509_STORE_CTX *csc;
+
+    STACK_OF(X509) *ustack;
+    STACK_OF(X509) *vstack;
+
+    // load certificates to temp structures
+
+    // first the end entity cert
+    // (EE cert needs to be passed separately to OpenSSL verification context)
+    f_name = hash_to_filename(cert_hashes[0]);
+    if (f_name == NULL)
+        return 1;
+
+    cert = load_cert(f_name);
+    free(f_name);
+    if (cert == NULL) {
+        printf("can't load certificate!\n");
+        return 1;
+    }
+
+    // then the intermediate certificates
+    ustack = sk_X509_new_null();
+
+    for (int i=1; cert_hashes[i]!=NULL; i++) {
+        //printf(".\n");
+        f_name = hash_to_filename(cert_hashes[i]);
+        if (f_name == NULL) {
+            // file not found
+            continue;
+        }
+        x509 = load_cert(f_name);
+        if (x509 == NULL) {
+            // loading cert failed
+            continue;
+        }
+        sk_X509_push(ustack, x509);
+        free(f_name);
+    }
+
+    // first try with just trusted certificates
+
+    store = SSL_CTX_get_cert_store(trusted_only);
+    if (store == NULL) {
+        fprintf(stderr, "store init failed\n");
+        return 1;
+    }
+    X509_STORE_set_flags(store, X509_V_FLAG_TRUSTED_FIRST);
+
+    csc = X509_STORE_CTX_new();
+
+    ret = X509_STORE_CTX_init(csc, store, cert, ustack);
+    if (ret != 1) {
+        return 1;
+    }
+
+    ret = X509_verify_cert(csc);
+
+    if (ret != 1) {
+       // printf("%s\n", X509_verify_cert_error_string(csc->error));
+    } else {
+        // chain is complete, output certificate hashes
+        printf("{\"chain\":\"complete\",\"certificates\":[");
+        vstack = X509_STORE_CTX_get_chain(csc);
+        for(int i=0; i<sk_X509_num(vstack); i++) {
+            X509 *c = sk_X509_value(vstack, i);
+
+            const EVP_MD *digest;
+            unsigned char md[EVP_MAX_MD_SIZE];
+            int n;
+            digest = EVP_get_digestbyname("sha256");
+            X509_digest(c, digest, md, &n);
+            printf("\"");
+            for(int i=0; i<n; i++) {
+                printf("%02x", md[i]);
+            }
+            printf("\"");
+            if (i+1 < sk_X509_num(vstack)) {
+                printf(",");
+            }
+        }
+        printf("]}");
+        X509_STORE_CTX_free(csc);
+        sk_X509_pop_free(ustack, X509_free);
+        X509_free(cert);
+        return 0;
+    }
+    X509_STORE_CTX_free(csc);
+
+    // validation failed with just the trust anchors, retry with all
+    // known intermediate certificates
+
+    store = SSL_CTX_get_cert_store(all_CAs);
+    if (store == NULL) {
+        fprintf(stderr, "store init failed\n");
+        return 1;
+    }
+    X509_STORE_set_flags(store, X509_V_FLAG_TRUSTED_FIRST);
+
+    csc = X509_STORE_CTX_new();
+
+    ret = X509_STORE_CTX_init(csc, store, cert, ustack);
+    if (ret != 1) {
+        return 1;
+    }
+
+    ret = X509_verify_cert(csc);
+    if (ret != 1) {
+        // certificate untrusted
+        printf("{\"chain\":\"untrusted\"}");
+    } else {
+        // chain successfully verified using all certificates,
+        // print all the certs used to verify it
+        printf("{\"chain\":\"incomplete\",\"certificates\":[");
+        vstack = X509_STORE_CTX_get_chain(csc);
+        for(int i=0; i<sk_X509_num(vstack); i++) {
+            X509 *c = sk_X509_value(vstack, i);
+
+            const EVP_MD *digest;
+            unsigned char md[EVP_MAX_MD_SIZE];
+            int n;
+            digest = EVP_get_digestbyname("sha256");
+            X509_digest(c, digest, md, &n);
+            printf("\"");
+            for(int i=0; i<n; i++) {
+                printf("%02x", md[i]);
+            }
+            printf("\"");
+            if (i+1 < sk_X509_num(vstack)) {
+                printf(",");
+            }
+        }
+        printf("]}");
+    }
+
+    X509_STORE_CTX_free(csc);
+    sk_X509_pop_free(ustack, X509_free);
+    X509_free(cert);
+
+    return 0;
+}
+
+// check if array of strings in json object is the same or not
+int string_array_cmp(struct json_object *a, struct json_object *b)
+{
+    if (json_object_get_type(a) != json_type_array)
+        return -1; // wrong type
+
+    if (json_object_get_type(b) != json_type_array)
+        return -1; // wrong type
+
+    if (json_object_array_length(a) != json_object_array_length(b))
+        return 1;
+
+    for (int i=0; i<json_object_array_length(a); i++) {
+        struct json_object *s_a, *s_b;
+        const char *str_a, *str_b;
+
+        s_a = json_object_array_get_idx(a, i);
+        if (json_object_get_type(s_a) != json_type_string)
+            return -1; // wrong type
+
+        s_b = json_object_array_get_idx(b, i);
+        if (json_object_get_type(s_b) != json_type_string)
+            return -1; // wrong type
+
+        str_a = json_object_get_string(s_a);
+        str_b = json_object_get_string(s_b);
+        if (str_a == NULL && str_b == NULL)
+            continue;
+
+        if (str_a == NULL || str_b == NULL)
+            return 1;
+
+        if (strcmp(str_a, str_b) != 0)
+            return 1;
+    }
+
+    return 0;
+}
+
+// add a list of new strings (hashes) to a list of known strings, if they are
+// indeed new, don't do anything if they are already in the known set
+int register_known_chains(struct json_object ***known, struct json_object *new)
+{
+    int rc;
+
+    if (*known == NULL) {
+        *known = calloc(sizeof(struct json_object**), 2);
+        (*known)[0] = new;
+        return 0; // it's a new one
+    }
+
+    int i;
+    for (i=0; (*known)[i] != NULL; i++) {
+        rc = string_array_cmp((*known)[i], new);
+        if (rc < 0) {
+            fprintf(stderr, "error in string_array_cmp\n");
+        }
+        if (string_array_cmp((*known)[i], new) == 0) {
+            return 1; // we've seen it before
+        }
+    }
+
+    // add it to known objects
+    *known = realloc(*known, sizeof(struct json_object **)*(i+2));
+    // TODO handle errors
+    (*known)[i] = new;
+    (*known)[i+1] = NULL;
+    return 0;
+}
+
+struct json_object *read_json_from_file(char *filename)
+{
+    json_tokener *tok;
+
+    struct json_object *obj = NULL;
+    int ret = 0;
+    int rc;
+    size_t len = 8192;
+    char buffer[len];
+    char *start;
+    int i;
+    enum json_tokener_error jerr;
+
+    int fd;
+
+    fd = open(filename, 0);
+    if (fd < 0) {
+        ret = 1;
+        goto err;
+    }
+    // skip garbage at the beginning of file (old `cipherscan` versions
+    // sometimes did put `popd` and pushd` output in the json file)
+    do {
+        rc = read(fd, buffer, 1);
+    } while (buffer[0] != '{' || rc < 0);
+    if (rc >= 0) {
+        lseek(fd, -1, SEEK_CUR);
+    }
+
+    // parse the json object from the file
+    tok = json_tokener_new();
+    do {
+        rc = read(fd, buffer, len);
+        if (rc < 0)
+            break;
+        obj = json_tokener_parse_ex(tok, buffer, rc);
+    } while ((jerr = json_tokener_get_error(tok)) == json_tokener_continue);
+
+    if (jerr != json_tokener_success){
+        fprintf(stderr, "error in file %s, line: %s\n", filename, buffer);
+    }
+
+tok_free:
+
+    json_tokener_free(tok);
+
+close_fd:
+    close(fd);
+
+err:
+    if (ret) {
+        fprintf(stderr, "error while reading file: %i", ret);
+    }
+    return obj;
+}
+
+// process all ciphersuites one by one from a given host results file
+int process_host_results(char *filename)
+{
+    int fd;
+    int ret = 0;
+    int rc;
+    size_t sz;
+    size_t alloc_size = 64 * 1024;
+    const char *str;
+    struct json_object *root;
+    struct json_object *ciphers;
+    struct json_object *current;
+    struct json_object *certificates;
+
+    struct json_object **known_chains;
+    known_chains = malloc(sizeof(struct json_object*) * 1);
+    known_chains[0] = NULL;
+
+    struct lh_table *table;
+    enum json_type obj_t;
+    json_bool j_rc;
+
+    root = read_json_from_file(filename);
+    if (root == NULL) {
+        ret = 1;
+        goto err;
+    }
+
+    obj_t = json_object_get_type(root);
+    str = json_type_to_name(obj_t);
+
+    j_rc = json_object_object_get_ex(root, "ciphersuite", &ciphers);
+    if (j_rc == FALSE) {
+        ret = 1;
+        goto json_free;
+    }
+
+    // ok, we've got the ciphersuite part, we can print the json header for
+    // the host file
+    printf("{\"host\":\"%s\",\"chains\":[", filename);
+
+    int first_printed=0;
+    for(int i=0; i < json_object_array_length(ciphers); i++) {
+        current = json_object_array_get_idx(ciphers, i);
+        //printf("\t[%i]:\n", i);
+        j_rc = json_object_object_get_ex(current, "certificates", &certificates);
+        if (j_rc == FALSE)
+            continue;
+
+        const char** certs;
+        certs = calloc(sizeof(const char*), json_object_array_length(certificates) + 1);
+        int j;
+        for (j=0; j < json_object_array_length(certificates); j++) {
+            certs[j] = json_object_get_string(json_object_array_get_idx(certificates, j));
+            //printf("\t\t\t%s\n", certs[j]);
+        }
+        rc = register_known_chains(&known_chains, certificates);
+        //printf("\t\t%i\n", rc);
+
+        if (rc == 0 && j > 0) {
+            if (first_printed != 0)
+                printf(",");
+            if (process_chain(certs) != 0) {
+                fprintf(stderr, "error while processing chains!\n");
+            } else {
+                first_printed = 1;
+            }
+        }
+
+        // DEBUG, print whole json "object" object
+        //json_object_object_foreach(current, key, val) {
+        //    str = json_object_to_json_string(val);
+        //    printf("\t\t%s: %s\n", key, str);
+        //}
+
+        free(certs);
+    }
+    printf("]}");
+
+json_free:
+    json_object_put(root);
+
+err:
+    free(known_chains);
+    return ret;
+}
+
+int main(int argc, char** argv)
+{
+    int ret;
+
+    DIR *dirp;
+    struct dirent *direntp;
+
+    char buffer[8192] = {};
+
+    SSL_load_error_strings();
+    SSL_library_init();
+
+    /* init trust stores with certificate locations */
+    trusted_only = SSL_CTX_new(SSLv23_method());
+    if (trusted_only == NULL) {
+        ERR_print_errors_fp(stderr);
+        return 1;
+    }
+
+    ret = SSL_CTX_load_verify_locations(trusted_only, NULL, CA_TRUSTED);
+    if (ret != 1) {
+        ERR_print_errors_fp(stderr);
+        return 1;
+    }
+
+    all_CAs = SSL_CTX_new(SSLv23_method());
+    if (all_CAs == NULL) {
+        ERR_print_errors_fp(stderr);
+        return 1;
+    }
+
+    ret = SSL_CTX_load_verify_locations(all_CAs, NULL, CA_ALL);
+    if (ret != 1) {
+        ERR_print_errors_fp(stderr);
+        return 1;
+    }
+
+    /* traverse the result directory, check all files in turn */
+    dirp=opendir("results");
+    while((direntp=readdir(dirp)) != NULL) {
+        if (strcmp(direntp->d_name, ".") == 0)
+            continue;
+        if (strcmp(direntp->d_name, "..") == 0)
+            continue;
+
+        snprintf(buffer, 8191, "results/%s", direntp->d_name);
+
+        ret = process_host_results(buffer);
+        if (ret == 1) {
+            fprintf(stderr, "error while processing %s\n", buffer);
+        }
+        if (ret == 0)
+            printf("\n");
+    }
+    closedir(dirp);
+
+    /* clean up */
+    SSL_CTX_free(trusted_only);
+    SSL_CTX_free(all_CAs);
+    all_CAs = NULL;
+    trusted_only = NULL;
+
+    return ret;
+}

top1m/parse_CAs.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Author: Hubert Kario - 2014
+
+from __future__ import division
+
+path = "./results/"
+ca_certs_path = "./ca_files"
+certs_path = "./certs"
+
+""" only root CAs, no cached intermediate certs """
+trust_path = "./ca_trusted"
+
+import json
+import sys
+from collections import defaultdict
+import os
+from OpenSSL import crypto
+
+invocations = defaultdict(int)
+
+total = 0
+hosts = 0
+chains = defaultdict(int)
+chain_len = defaultdict(int)
+keysize = defaultdict(int)
+keysize_per_chain = defaultdict(int)
+root_CA = defaultdict(int)
+sig_alg = defaultdict(int)
+intermediate_CA = defaultdict(int)
+effective_security = defaultdict(int)
+
+subject_hashes = {}
+issuer_hashes = {}
+
+def get_cert_subject_name(cert):
+    subject = cert.get_subject()
+    commonName = None
+    organization = None
+
+    for elem,val in subject.get_components():
+        if elem == "CN" and commonName is None:
+            commonName = val
+        if elem == "O" and organization is None:
+            organization = val
+
+    s_hash = "(" + ("%0.8X" % subject.hash()).lower() + ") "
+
+    if commonName is not None:
+        return s_hash + commonName
+    elif organization is not None:
+        return s_hash + organization
+    else:
+        return s_hash
+
+def get_path_for_hash(cert_hash):
+    f_name = certs_path + '/' + cert_hash + '.pem'
+    if not os.path.exists(f_name):
+        f_name = ca_certs_path + '/' + cert_hash + '.pem'
+        if not os.path.exists(f_name):
+            #print "File with hash " + c_hash + " is missing!"
+            return None
+    return f_name
+
+""" convert RSA and DSA key sizes to estimated Level of security """
+def rsa_key_size_to_los(size):
+    if size < 760:
+        return 40
+    elif size < 1020:
+        return 64
+    elif size < 2040:
+        return 80
+    elif size < 3068:
+        return 112
+    elif size < 4094:
+        return 128
+    elif size < 7660:
+        return 152
+    elif size < 15300:
+        return 192
+    else:
+        return 256
+
+def sig_alg_to_los(name):
+    if 'MD5' in name.upper():
+        return 64
+    elif 'SHA1' in name.upper():
+        return 80
+    elif 'SHA224' in name.upper():
+        return 112
+    elif 'SHA256' in name.upper():
+        return 128
+    elif 'SHA384' in name.upper():
+        return 192
+    elif 'SHA512' in name.upper():
+        return 256
+    else:
+        raise UnknownSigAlgError
+
+def collect_key_sizes(file_names):
+
+    tmp_keysize = {}
+
+    """ don't collect signature alg for the self signed root """
+    with open(file_names[-1]) as cert_file:
+        cert_pem = cert_file.read()
+
+    cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+
+    pubkey = cert.get_pubkey()
+    if pubkey.type() == crypto.TYPE_RSA:
+        keysize['RSA ' + str(pubkey.bits())] += 1
+        tmp_keysize['RSA ' + str(pubkey.bits())] = 1
+        security_level = rsa_key_size_to_los(pubkey.bits())
+    elif pubkey.type() == crypto.TYPE_DSA:
+        keysize['DSA ' + str(pubkey.bits())] += 1
+        tmp_keysize['DSA ' + str(pubkey.bits())] = 1
+        security_level = rsa_key_size_to_los(pubkey.bits())
+    elif pubkey.type() == 408:
+        keysize['ECDSA ' + str(pubkey.bits())] += 1
+        tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1
+        security_level = pubkey.bits()/2
+    else:
+        keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1
+        security_level = 0
+
+    root_CA[get_cert_subject_name(cert)] += 1
+
+    """ exclude the self signed root and server cert from stats """
+    for f_name in file_names[1:-1]:
+        with open(f_name) as cert_file:
+            cert_pem = cert_file.read()
+
+        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+
+        pubkey = cert.get_pubkey()
+        if pubkey.type() == crypto.TYPE_RSA:
+            keysize['RSA ' + str(pubkey.bits())] += 1
+            tmp_keysize['RSA ' + str(pubkey.bits())] = 1
+            c_key_level = rsa_key_size_to_los(pubkey.bits())
+        elif pubkey.type() == crypto.TYPE_DSA:
+            keysize['DSA ' + str(pubkey.bits())] += 1
+            tmp_keysize['DSA ' + str(pubkey.bits())] = 1
+            c_key_level = rsa_key_size_to_los(pubkey.bits())
+        elif pubkey.type() == 408:
+            keysize['ECDSA ' + str(pubkey.bits())] += 1
+            tmp_keysize['ECDSA ' + str(pubkey.bits())] = 1
+            c_key_level = pubkey.bits() / 2
+        else:
+            keysize[str(pubkey.type()) + ' ' + str(pubkey.bits())] += 1
+            c_key_level = 0
+
+        if security_level > c_key_level:
+            security_level = c_key_level
+
+        sig_alg[cert.get_signature_algorithm()] += 1
+        c_sig_level = sig_alg_to_los(cert.get_signature_algorithm())
+        if security_level > c_sig_level:
+            security_level = c_sig_level
+
+        intermediate_CA[get_cert_subject_name(cert)] += 1
+
+    for key_s in tmp_keysize:
+        keysize_per_chain[key_s] += 1
+
+    # XXX doesn't handle the situation in which the CA uses its certificate
+    # for a web server properly
+    with open(file_names[0]) as cert_file:
+       cert_pem = cert_file.read()
+
+    cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
+
+    pubkey = cert.get_pubkey()
+    if pubkey.type() == crypto.TYPE_RSA:
+       c_key_level = rsa_key_size_to_los(pubkey.bits())
+    elif pubkey.type() == crypto.TYPE_DSA:
+       c_key_level = rsa_key_size_to_los(pubkey.bits())
+    elif pubkey.type() == 408:
+       c_key_level = pubkey.bits() / 2
+    else:
+       c_key_level = 0
+
+    if security_level > c_key_level:
+       security_level = c_key_level
+
+    c_sig_level = sig_alg_to_los(cert.get_signature_algorithm())
+    if security_level > c_sig_level:
+       security_level = c_sig_level
+
+    effective_security[security_level] += 1
+
+
+with open("parsed") as res_file:
+    for line in res_file:
+        try:
+            res = json.loads(line)
+        except ValueError as e:
+            print "can't process line: " + line
+            continue
+
+        f=res
+
+        try:
+            server_chain_trusted = False
+            server_chain_complete = False
+            server_chains = []
+            valid = False
+
+            """ Keep certificates in memory for a given file """
+            known_certs = {}
+
+            if not "chains" in f:
+                continue
+
+            results = f["chains"]
+
+            """ discard hosts with empty results """
+            if len(results) < 1:
+                continue
+
+            """ loop over list of ciphers """
+            for entry in results:
+
+                """ skip invalid results """
+                if not 'chain' in entry:
+                    continue
+
+                valid = True
+
+                if entry['chain'] == "untrusted":
+                    continue
+
+                if entry['chain'] == "complete":
+                    server_chain_complete = True
+                    server_chain_trusted = True
+
+                if entry['chain'] == "incomplete":
+                    server_chain_trusted = True
+
+                server_chains += [entry['certificates']]
+
+            if server_chain_trusted:
+                if server_chain_complete:
+                    chains["complete"] += 1
+                    print "complete: " + f['host']
+                else:
+                    chains["incomplete"] += 1
+                    print "incomplete: " + f['host']
+            else:
+                chains["untrusted"] += 1
+                print "untrusted: " + f['host']
+
+            if valid:
+                hosts += 1
+
+            for chain in server_chains:
+                f_names = []
+                for hash in chain:
+                    path = get_path_for_hash(hash)
+                    f_names += [path]
+
+                collect_key_sizes(f_names)
+                chain_len[str(len(chain))] += 1
+                if len(chain) == 1:
+                    sys.stderr.write("file with chain 1 long: " + line)
+                total += 1
+        except TypeError as e:
+
+            sys.stderr.write("can't process: " + line)
+            continue
+
+""" Display stats """
+#print "openssl invocations: " + str(invocations["openssl"])
+
+print "Statistics from " + str(total) + " chains provided by " + str(hosts) + " hosts"
+
+print("\nServer provided chains    Count     Percent")
+print("-------------------------+---------+-------")
+for stat in sorted(chains):
+    percent = round(chains[stat] / hosts * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(chains[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nTrusted chain statistics")
+print("========================")
+
+
+print("\nChain length              Count     Percent")
+print("-------------------------+---------+-------")
+for stat in sorted(chain_len):
+    percent = round(chain_len[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(chain_len[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nCA key size in chains     Count")
+print("-------------------------+---------")
+for stat in sorted(keysize):
+    sys.stdout.write(stat.ljust(25) + " " + str(keysize[stat]).ljust(10) + "\n")
+
+print("\nChains with CA key        Count     Percent")
+print("-------------------------+---------+-------")
+for stat in sorted(keysize_per_chain):
+    percent = round(keysize_per_chain[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(keysize_per_chain[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nSignature algorithm (ex. root) Count")
+print("------------------------------+---------")
+for stat in sorted(sig_alg):
+    sys.stdout.write(stat.ljust(30) + " " + str(sig_alg[stat]).ljust(10) + "\n")
+
+print("\nEff. host cert chain LoS  Count     Percent")
+print("-------------------------+---------+-------")
+for stat in sorted(effective_security):
+    percent = round(effective_security[stat] / total * 100, 4)
+    sys.stdout.write(str(stat).ljust(25) + " " + str(effective_security[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nRoot CAs                                      Count     Percent")
+print("---------------------------------------------+---------+-------")
+for stat in sorted(root_CA):
+    percent = round(root_CA[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(45)[0:45] + " " + str(root_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nIntermediate CA                               Count     Percent")
+print("---------------------------------------------+---------+-------")
+for stat in sorted(intermediate_CA):
+    percent = round(intermediate_CA[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(45)[0:45] + " " + str(intermediate_CA[stat]).ljust(10) + str(percent).ljust(4) + "\n")

top1m/parse_results.py

@@ -14,10 +14,47 @@ import json
 import sys
 from collections import defaultdict
 import os
+import re
+
+def natural_sort(l):
+    convert = lambda text: int(text) if text.isdigit() else text.lower()
+    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
+    return sorted(l, key = alphanum_key)
+
+""" list of ciphers offerred by Firefox 29 by default """
+firefox_ciphers=[
+        'ECDHE-ECDSA-AES128-GCM-SHA256',
+        'ECDHE-RSA-AES128-GCM-SHA256',
+        'ECDHE-ECDSA-AES256-SHA',
+        'ECDHE-ECDSA-AES128-SHA',
+        'ECDHE-RSA-AES128-SHA',
+        'ECDHE-RSA-AES256-SHA',
+        'ECDHE-RSA-DES-CBC3-SHA',
+        'ECDHE-ECDSA-RC4-SHA',
+        'ECDHE-RSA-RC4-SHA',
+        'DHE-RSA-AES128-SHA',
+        'DHE-DSS-AES128-SHA',
+        'DHE-RSA-CAMELLIA128-SHA',
+        'DHE-RSA-AES256-SHA',
+        'DHE-DSS-AES256-SHA',
+        'DHE-RSA-CAMELLIA256-SHA',
+        'EDH-RSA-DES-CBC3-SHA',
+        'AES128-SHA',
+        'CAMELLIA128-SHA',
+        'AES256-SHA',
+        'CAMELLIA256-SHA',
+        'DES-CBC3-SHA',
+        'RC4-SHA',
+        'RC4-MD5']
 
 report_untrused=False
 
 cipherstats = defaultdict(int)
+FF_RC4_Only_cipherstats = defaultdict(int)
+FF_RC4_preferred_cipherstats = defaultdict(int)
+FF_incompatible_cipherstats = defaultdict(int)
+FF_selected_cipherstats = defaultdict(int)
+cipherordering = defaultdict(int)
 pfsstats = defaultdict(int)
 protocolstats = defaultdict(int)
 handshakestats = defaultdict(int)
@@ -38,6 +75,7 @@ for r,d,flist in os.walk(path):
         tempdsakeystats = {}
         tempsigstats = {}
         tempticketstats = {}
+        """ supported ciphers by the server under scan """
         tempcipherstats = {}
         ciphertypes = 0
         AESGCM = False
@@ -46,6 +84,13 @@ for r,d,flist in os.walk(path):
         DES3 = False
         CAMELLIA = False
         RC4 = False
+        """ the following depends on FF_compat, so by default it can be True """
+        RC4_Only_FF = True
+        FF_compat = False
+        temp_FF_incompat = {}
+        list_of_ciphers = []
+        FF_RC4_Pref = None
+        FF_selected = None
         ADH = False
         DHE = False
         AECDH = False
@@ -85,11 +130,25 @@ for r,d,flist in os.walk(path):
                 if 'False' in entry['trusted'] and report_untrused == False:
                     continue
 
+                list_of_ciphers.append(entry['cipher'])
+
+                # check if the advertised ciphers are not effectively RC4 Only
+                # for firefox or incompatible with firefox
+                if entry['cipher'] in firefox_ciphers:
+                    # if this is first cipher and we already are getting RC4
+                    # then it means that RC4 is preferred
+                    FF_compat = True
+                    if not 'RC4' in entry['cipher']:
+                        RC4_Only_FF = False
+                else:
+                    temp_FF_incompat[entry['cipher']] = 1
+
                 """ store the ciphers supported """
                 if 'ADH' in entry['cipher'] or 'AECDH' in entry['cipher']:
                     ciphertypes += 1
                     name = "z:" + entry['cipher']
                     tempcipherstats[name] = 1
+                    tempcipherstats['Insecure'] = 1
                 elif 'AES128-GCM' in entry['cipher'] or 'AES256-GCM' in entry['cipher']:
                     if not AESGCM:
                         AESGCM = True
@@ -114,10 +173,15 @@ for r,d,flist in os.walk(path):
                     if not CHACHA20:
                         ciphertypes += 1
                         CHACHA20 = True
+                elif 'IDEA' in entry['cipher'] or 'SEED' in entry['cipher']:
+                    ciphertypes += 1
+                    name = "y:" + entry['cipher']
+                    tempcipherstats[name] = 1
                 else:
                     ciphertypes += 1
                     name = "z:" + entry['cipher']
                     tempcipherstats[name] = 1
+                    tempcipherstats['Insecure'] = 1
 
                 """ store key handshake methods """
                 if 'ECDHE' in entry['cipher']:
@@ -138,7 +202,7 @@ for r,d,flist in os.walk(path):
                     RSA = True
 
                 """ save the key size """
-                if 'ECDSA' in entry['cipher']:
+                if 'ECDSA' in entry['cipher'] or 'ECDH-RSA' in entry['cipher']:
                     ECDSA = True
                     tempecckeystats[entry['pubkey'][0]] = 1
                 elif 'DSS' in entry['cipher']:
@@ -206,6 +270,32 @@ for r,d,flist in os.walk(path):
         if dualstack:
             dsarsastack += 1
 
+        """ save cipher ordering """
+        if 'serverside' in results:
+            if results['serverside'] == "False":
+                cipherordering['Client side'] += 1
+            else:
+                cipherordering['Server side'] += 1
+        else:
+            cipherordering['Unknown'] += 1
+
+        """ simulate handshake with Firefox """
+        if FF_compat:
+            if 'serverside' in results and results['serverside'] == "False":
+                for cipher in firefox_ciphers:
+                    if cipher in list_of_ciphers:
+                        FF_selected = cipher
+                        if 'RC4' in cipher:
+                            FF_RC4_Pref = True
+                        break
+            else:
+                for cipher in list_of_ciphers:
+                    if cipher in firefox_ciphers:
+                        FF_selected = cipher
+                        if 'RC4' in cipher:
+                            FF_RC4_Pref = True
+                        break
+
         for s in tempsigstats:
             sigalg[s] += 1
 
@@ -256,6 +346,27 @@ for r,d,flist in os.walk(path):
                         cipherstats['RC4 forced in TLS1.1+'] += 1
                 cipherstats['RC4 Preferred'] += 1
 
+        if FF_compat:
+            if 'ECDHE' in FF_selected:
+                FF_selected_cipherstats['x:ECDHE'] += 1
+            elif 'DHE' in FF_selected or 'EDH' in FF_selected:
+                FF_selected_cipherstats['x:DHE'] += 1
+            else:
+                FF_selected_cipherstats['x:kRSA'] += 1
+            FF_selected_cipherstats[FF_selected] += 1
+            if RC4_Only_FF and ciphertypes != 1:
+                cipherstats['x:FF 29 RC4 Only'] += 1
+                for cipher in temp_FF_incompat:
+                    FF_RC4_Only_cipherstats[cipher] += 1
+            if FF_RC4_Pref and not 'RC4' in results['ciphersuite'][0]['cipher']:
+                cipherstats['x:FF 29 RC4 Preferred'] += 1
+                for cipher in temp_FF_incompat:
+                    FF_RC4_preferred_cipherstats[cipher] += 1
+        else:
+            cipherstats['x:FF 29 incompatible'] += 1
+            for cipher in temp_FF_incompat:
+                FF_incompatible_cipherstats[cipher] += 1
+
         for cipher in tempcipherstats:
             cipherstats[cipher] += 1
 
@@ -309,6 +420,13 @@ for r,d,flist in os.walk(path):
     #if total % 1999 == 0:
     #    break
 
+""" The 'x:FF 29 RC4 Preferred' counts only sites that effectively prefer
+    RC4 when using FF, to make reporting more readable, sum it with sites
+    that do that for all ciphers"""
+
+if "x:FF 29 RC4 Preferred" in cipherstats and "RC4 Preferred" in cipherstats:
+    cipherstats['x:FF 29 RC4 Preferred'] += cipherstats['RC4 Preferred']
+
 print("SSL/TLS survey of %i websites from Alexa's top 1 million" % total)
 if report_untrused == False:
     print("Stats only from connections that did provide valid certificates")
@@ -321,6 +439,36 @@ for stat in sorted(cipherstats):
     percent = round(cipherstats[stat] / total * 100, 4)
     sys.stdout.write(stat.ljust(25) + " " + str(cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
 
+print("\nCipher ordering           Count     Percent")
+print("-------------------------+---------+-------")
+for stat in sorted(cipherordering):
+    percent = round(cipherordering[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(25) + " " + str(cipherordering[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nFF 29 selected ciphers        Count    Percent")
+print("-----------------------------+---------+------")
+for stat in sorted(FF_selected_cipherstats):
+    percent = round(FF_selected_cipherstats[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(30) + " " + str(FF_selected_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nFF 29 RC4 Only other ciphers  Count    Percent")
+print("-----------------------------+---------+------")
+for stat in sorted(FF_RC4_Only_cipherstats):
+    percent = round(FF_RC4_Only_cipherstats[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_Only_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nFF 29 RC4 pref other ciphers  Count    Percent")
+print("-----------------------------+---------+------")
+for stat in sorted(FF_RC4_preferred_cipherstats):
+    percent = round(FF_RC4_preferred_cipherstats[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(30) + " " + str(FF_RC4_preferred_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
+print("\nFF 29 incompatible ciphers    Count    Percent")
+print("-----------------------------+---------+------")
+for stat in sorted(FF_incompatible_cipherstats):
+    percent = round(FF_incompatible_cipherstats[stat] / total * 100, 4)
+    sys.stdout.write(stat.ljust(30) + " " + str(FF_incompatible_cipherstats[stat]).ljust(10) + str(percent).ljust(4) + "\n")
+
 print("\nSupported Handshakes      Count     Percent")
 print("-------------------------+---------+-------")
 for stat in sorted(handshakestats):
@@ -340,7 +488,7 @@ for stat in sorted(pfsstats):
 
 print("\nTLS session ticket hint   Count     Percent ")
 print("-------------------------+---------+--------")
-for stat in sorted(tickethint):
+for stat in natural_sort(tickethint):
     percent = round(tickethint[stat] / total * 100, 4)
     sys.stdout.write(stat.ljust(25) + " " + str(tickethint[stat]).ljust(10) + str(percent).ljust(9) + "\n")
 

top1m/process-certificate-statistics.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+if [ ! -d ./ca_files ]; then
+    echo "Directory with collected CA certificates missing!" >&2
+    exit 1
+fi
+
+if [ ! -d ./ca_trusted ]; then
+    echo "Directory with just trust anchors missing!" >&2
+    exit 1
+fi
+
+if [ ! -d ./certs ]; then
+    echo "Directory with certificates missing!" >&2
+    exit 1
+fi
+
+if ! ls -f ./ca_files/????????.? > /dev/null; then
+    echo "CA certificates directory not hashed properly (use c_rehash)" >&2
+    exit 1
+fi
+
+if ! ls -f ./ca_trusted/????????.? > /dev/null; then
+    echo "Directory with trust anchors not hashed properly (use c_rehash)" >&2
+    exit 1
+fi
+
+if [ ! -d ./results ]; then
+    echo "Directory with scan results missing!" >&2
+    exit 1
+fi
+
+if [ ! -x ./parse_CAs ]; then
+    echo "Compiling parse_CAs script"
+    gcc -o parse_CAs parse_CAs.c -lssl -lcrypto -ljson-c --std=gnu99
+    if [ $? -ne 0 ]; then
+        echo "Compilation failed, aborting" >&2
+        exit 1
+    fi
+fi
+
+echo "Verifying certificate chains from results files"
+./parse_CAs > parsed
+echo "Calculating statistics for verified certificate chains"
+python parse_CAs.py > trust_scan
+echo "Done!"
+echo "Results are in \"trust_scan\" file"

top1m/testtop1m.sh

@@ -2,20 +2,44 @@
 parallel=10
 max_bg=50
 absolute_max_bg=100
-max_load=50
+max_load_avg=50
 
 if [ $(ulimit -u) -lt $((10*absolute_max_bg)) ]; then
     echo "max user processes too low, use ulimit -u to increase"
     exit 1
 fi
 [ ! -e "results" ] && mkdir results
+[ ! -e "certs" ] && mkdir certs
+if [ ! -e "ca_files" ]; then
+    mkdir ca_files
+    pushd ca_files >/dev/null
+    awk '
+      split_after == 1 {n++;split_after=0}
+      /-----END CERTIFICATE-----/ {split_after=1}
+      {print > "cert" n ".pem"}' < "/etc/pki/tls/certs/ca-bundle.crt"
+    for i in *; do
+        h=$(../../openssl x509 -hash -noout -in "$i" 2>/dev/null)
+        for num in `seq 0 100`; do
+            if [[ $h.$num -ef $i ]]; then
+                # file already linked, ignore
+                break
+            fi
+            if [[ ! -e $h.$num ]]; then
+                # file doesn't exist, create a link
+                ln -s "$i" "$h.$num"
+                break
+            fi
+        done
+    done
+    popd >/dev/null
+fi
 
 function wait_for_jobs() {
     local no_jobs
     no_jobs=$(jobs | wc -l)
 
-    while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do
+    while [ $no_jobs -gt $1 ] || awk -v maxload=$max_load_avg '{ if ($1 < maxload) exit 1 }' /proc/loadavg; do
-        if awk -v maxload=$max_load '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then
+        if awk -v maxload=$max_load_avg '{ if ($1 > maxload) exit 1 }' /proc/loadavg && [ $no_jobs -lt $absolute_max_bg ]; then
             return
         fi
         sleep 1
@@ -32,7 +56,7 @@ function scan_host() {
     if [ $? -gt 0 ]; then
         return
     fi
-    ../cipherscan -json -servername $1 $2:443 > results/$1@$2
+    ../cipherscan --capath ca_files --savecrt certs -json -servername $1 $2:443 > results/$1@$2
 }
 
 function scan_host_no_sni() {
@@ -44,10 +68,12 @@ function scan_host_no_sni() {
     if [ $? -gt 0 ]; then
         return
     fi
-    ../cipherscan -json $1:443 > results/$1
+    ../cipherscan --capath ca_files --savecrt certs -json $1:443 > results/$1
 }
 
 function scan_hostname() {
+    # check if the hostname isn't an IP address (since we can't put IP
+    # addresses to SNI extension)
     if [[ ! -z $(awk -F. '$1>=0 && $1<=255 && $2>=0 && $2<=255 &&
         $3>=0 && $3<=255 && $4>=0 && $4<=255 && NF==4' <<<"$1") ]]; then
         scan_host_no_sni $1