From 7a92186122e83f556487331d384a5811dc7cd45a Mon Sep 17 00:00:00 2001
From: Hubert Kario <hkario@redhat.com>
Date: Fri, 4 Apr 2014 20:12:50 +0200
Subject: [PATCH] Improve scanning performance and reduce false negatives

scan all the machines from top-1m.csv file, wait for completion
of all jobs

i=1 is an off-by-one-error

support top-1m.csv files with arbitrary number of sites

run scans for many hosts at a time, but don't run more than
specified amount

in case where default domain name doesn't resolve or doesn't have
port 443 open, retry with www. prefix
---
 top1m/testtop1m.sh | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/top1m/testtop1m.sh b/top1m/testtop1m.sh
index 52f504b..71b1b0f 100755
--- a/top1m/testtop1m.sh
+++ b/top1m/testtop1m.sh
@@ -1,14 +1,37 @@
 #!/usr/bin/env bash
 parallel=50
+max_bg=400
 [ ! -e "results" ] && mkdir results
-i=1
-while [ $i -lt 1000000 ]
+
+function wait_for_jobs() {
+    local no_jobs
+    no_jobs=$(jobs | wc -l)
+
+    while [ $no_jobs -gt $1 ]; do
+        sleep 1
+        no_jobs=$(jobs | wc -l)
+    done
+}
+
+i=0
+count=$(wc -l top-1m.csv | awk '{print $1}')
+while [ $i -lt $count ]
 do
     echo processings sites $i to $((i + parallel))
-    for t in $(tail -$((1000000 - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2)
+    for t in $(tail -$(($count - $i)) top-1m.csv | head -$parallel |cut -d ',' -f 2)
     do
-        (tcping -u 10000000 $t 443; if [ $? -gt 0 ];then continue;fi;../cipherscan $t:443 -json > results/$t )&
+        (tcping -u 10000000 $t 443;
+         if [ $? -gt 0 ];then 
+             tcping -u 10000000 www.$t 443; 
+             if [ $? -gt 0 ]; then 
+                 continue; 
+             else 
+                 ../cipherscan -json www.$t:443 > results/www.$t
+                 continue;
+             fi;
+         fi;../cipherscan -json $t:443 > results/$t )&
     done
-    sleep 7
     i=$(( i + parallel))
+    wait_for_jobs $max_bg
 done
+wait