size intolerance checks

since TLSv1.3 client hello's will include initial client key share,
verify that the server is able to accept and process big client hello
messages

cscan.py

@@ -20,6 +20,7 @@ from cscan.config import Xmas_tree, IE_6, IE_8_Win_XP, \
 from cscan.modifiers import no_sni, set_hello_version, set_record_version, \
         no_extensions, truncate_ciphers_to_size, append_ciphers_to_size, \
         extend_with_ext_to_size, add_empty_ext
+from cscan.bisector import Bisect
 
 
 def scan_with_config(host, port, conf, hostname, __sentry=None, __cache={}):
@@ -291,6 +292,82 @@ def scan_TLS_intolerancies(host, port, hostname):
                                                     conf.extensions and not
                                                     conf.ssl2))
 
+    #for name in ["Xmas tree", "Huge Cipher List",
+    #             "Huge Cipher List (trunc c/16388)"]:
+    #    intolerancies[name] = all(conf_iterator(lambda conf:
+    #                                            conf.name == name))
+
+    def test_cb(client_hello):
+        ret = scan_with_config(host, port, lambda _:client_hello, hostname)
+        return simple_inspector(ret)
+
+    # most size intolerancies lie between 16385 and 16389 so short-circuit to
+    # them if possible
+    good_conf = next((configs[name] for name, result in results.items()
+                      if simple_inspector(result)), None)
+
+    if good_conf:
+        size_c_16382 = simple_inspector(scan_with_config(host, port,
+            append_ciphers_to_size(copy.deepcopy(good_conf), 16382), hostname))
+        size_c_16392 = simple_inspector(scan_with_config(host, port,
+            append_ciphers_to_size(copy.deepcopy(good_conf), 16392), hostname))
+
+        if size_c_16382 and not size_c_16392:
+            good = append_ciphers_to_size(copy.deepcopy(good_conf), 16382)
+            bad = append_ciphers_to_size(copy.deepcopy(good_conf), 16392)
+        elif not size_c_16382:
+            good = good_conf
+            bad = append_ciphers_to_size(copy.deepcopy(good_conf), 16382)
+        else:
+            bad = append_ciphers_to_size(copy.deepcopy(good_conf), 65536)
+            size_c_65536 = simple_inspector(scan_with_config(host, port,
+                bad, hostname))
+            if not size_c_65536:
+                good = None
+                intolerancies["size c/65536"] = False
+            else:
+                good = append_ciphers_to_size(copy.deepcopy(good_conf), 16392)
+
+        if good:
+            bisect = Bisect(good, bad, hostname, test_cb)
+            good_h, bad_h = bisect.run()
+            intolerancies["size c/{0}".format(len(bad_h.write()))] = True
+            intolerancies["size c/{0}".format(len(good_h.write()))] = False
+
+    # test extension size intolerance, again, most lie between 16385
+    # and 16389 so short-circuit if possible
+    good_conf = next((configs[name] for name, result in results.items()
+                      if configs[name].extensions and
+                      simple_inspector(result)), None)
+
+    if good_conf:
+        size_e_16382 = simple_inspector(scan_with_config(host, port,
+            extend_with_ext_to_size(copy.deepcopy(good_conf), 16382), hostname))
+        size_e_16392 = simple_inspector(scan_with_config(host, port,
+            extend_with_ext_to_size(copy.deepcopy(good_conf), 16392), hostname))
+
+        if size_e_16382 and not size_e_16392:
+            good = extend_with_ext_to_size(copy.deepcopy(good_conf), 16382)
+            bad = extend_with_ext_to_size(copy.deepcopy(good_conf), 16392)
+        elif not size_e_16382:
+            good = good_conf
+            bad = extend_with_ext_to_size(copy.deepcopy(good_conf), 16382)
+        else:
+            bad = extend_with_ext_to_size(copy.deepcopy(good_conf), 65536)
+            size_e_65536 = simple_inspector(scan_with_config(host, port,
+                bad, hostname))
+            if not size_e_65536:
+                good = None
+                intolerancies["size e/65536"] = False
+            else:
+                good = extend_with_ext_to_size(copy.deepcopy(good_conf), 16392)
+
+        if good:
+            bisect = Bisect(good, bad, hostname, test_cb)
+            good_h, bad_h = bisect.run()
+            intolerancies["size e/{0}".format(len(bad_h.write()))] = True
+            intolerancies["size e/{0}".format(len(good_h.write()))] = False
+
     if json_out:
         print(json.dumps(intolerancies))
     else:

cscan/bisector.py

@@ -0,0 +1,172 @@
+# Copyright (c) 2015 Hubert Kario <hkario@redhat.com>
+# Released under Mozilla Public License Version 2.0
+
+"""Find an itolerance through bisecting Client Hello"""
+
+import copy
+from tlslite.extensions import PaddingExtension
+
+def list_union(first, second):
+    """Return an union between two lists, preserving order"""
+    first_i = iter(first)
+    second_i = iter(second)
+    first_s = set(first)
+    second_s = set(second)
+
+    ret = []
+    first_el = next(first_i, None)
+    second_el = next(second_i, None)
+    while first_el is not None and second_el is not None:
+        if first_el != second_el:
+            if first_el in second_s and second_el in first_s:
+                # the second list is longer, so take from it
+                ret.append(second_el)
+                # no discard as we would have duplicates
+                second_el = next(second_i, None)
+                continue
+            if first_el not in second_s:
+                ret.append(first_el)
+                first_s.discard(first_el)
+                first_el = next(first_i, None)
+            if second_el not in first_s:
+                ret.append(second_el)
+                second_s.discard(second_el)
+                second_el = next(second_i, None)
+        else:
+            ret.append(first_el)
+            first_s.discard(first_el)
+            second_s.discard(first_el)
+            first_el = next(first_i, None)
+            second_el = next(second_i, None)
+    while first_el:
+        if first_el not in second_s:
+            ret.append(first_el)
+        first_el = next(first_i, None)
+    while second_el:
+        if second_el not in first_s:
+            ret.append(second_el)
+        second_el = next(second_i, None)
+    return ret
+
+
+def bisect_lists(first, second):
+    """Return a list that is in the "middle" between the given ones"""
+    # handle None special cases
+    if first is None and second is None:
+        return None
+    if first is not None and second is None:
+        first, second = second, first
+    if first is None and second is not None:
+        if len(second) == 0:
+            return None
+        elif len(second) == 1:
+            return []
+        else:
+            first = []
+    # make the second lists always the longer one
+    if len(first) > len(second):
+        second, first = first, second
+    first_s = set(first)
+    second_s = set(second)
+    union = list_union(first, second)
+    symmetric_diff = first_s.symmetric_difference(second_s)
+    # preserve order for the difference
+    symmetric_diff = [x for x in union if x in symmetric_diff]
+    half_diff = set(symmetric_diff[:len(symmetric_diff)//2])
+    intersection = first_s & second_s
+
+    return [x for x in union if x in half_diff or x in intersection]
+
+
+def bisect_padding_extension(first, second):
+    if first is None and second is None:
+        return None
+    if first is not None and second is None:
+        first, second = second, first
+    if first is None and second is not None:
+        if len(second.paddingData) == 0:
+            return None
+        elif len(second.paddingData) == 1:
+            return PaddingExtension()
+        else:
+            first = PaddingExtension()
+    return PaddingExtension().create((len(first.paddingData) +
+                                      len(second.paddingData)) // 2)
+
+
+def bisect_extensions(first, second):
+    # handle padding extension
+    if first is None and second is None:
+        return None
+    if first is not None and second is None:
+        first, second = second, first
+    if first is None and second is not None:
+        if len(second) == 0:
+            return None
+        if len(second) == 1:
+            return []
+        first = []
+    f_ext = next((x for x in first if isinstance(x, PaddingExtension)), None)
+    s_ext = next((x for x in second if isinstance(x, PaddingExtension)), None)
+
+    ext = bisect_padding_extension(f_ext, s_ext)
+    if ext is None:
+        # remove the extension
+        return [x for x in first if not isinstance(x, PaddingExtension)]
+    else:
+        if f_ext is None:
+            return first + [ext]
+        # replace extension
+        return [ext if isinstance(x, PaddingExtension) else x for x in first]
+
+
+def bisect_hellos(first, second):
+    """Return a client hello that is in the "middle" of two other"""
+    ret = copy.copy(first)
+
+    ret.client_version = ((first.client_version[0] + second.client_version[0])
+                          // 2,
+                          (first.client_version[1] + second.client_version[1])
+                          // 2)
+    ret.cipher_suites = bisect_lists(first.cipher_suites, second.cipher_suites)
+    ret.extensions = bisect_lists(first.extensions, second.extensions)
+    ret.compression_methods = bisect_lists(first.compression_methods,
+                                           second.compression_methods)
+    if first.extensions == ret.extensions \
+            or second.extensions == ret.extensions:
+        ret.extensions = bisect_extensions(first.extensions,
+                                           second.extensions)
+    return ret
+
+class Bisect(object):
+    """
+    Perform a bisection between two Client Hello's to find intolerance
+
+    Tries to find a cause for intolerance by using a bisection-like
+    algorithm
+    """
+
+    def __init__(self, good, bad, hostname, callback):
+        """Set the generators for good and bad hello's and callback to test"""
+        self.good = good
+        self.bad = bad
+        if hostname is not None:
+            self.hostname = bytearray(hostname, 'utf-8')
+        else:
+            self.hostname = None
+        self.callback = callback
+
+    def run(self):
+        good_hello = self.good(self.hostname)
+        bad_hello = self.bad(self.hostname)
+        middle = bisect_hellos(good_hello, bad_hello)
+
+        while good_hello != middle and \
+                middle != bad_hello:
+            if self.callback(middle):
+                good_hello = middle
+            else:
+                bad_hello = middle
+            middle = bisect_hellos(good_hello, bad_hello)
+
+        return (good_hello, bad_hello)

cscan_tests/test_bisector.py

@@ -0,0 +1,258 @@
+# Copyright (c) 2015 Hubert Kario
+# Released under Mozilla Public License Version 2.0
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+from tlslite.extensions import SignatureAlgorithmsExtension, SNIExtension, \
+        SupportedGroupsExtension, ECPointFormatsExtension, TLSExtension
+from cscan.config import HugeCipherList, VeryCompatible, IE_8_Win_XP
+from cscan.bisector import bisect_lists, list_union, Bisect
+from cscan.modifiers import extend_with_ext_to_size, append_ciphers_to_size
+
+class TestListUnion(unittest.TestCase):
+    def test_identical(self):
+        a = [1, 2, 3, 4]
+        b = [1, 2, 3, 4]
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 2, 3, 4])
+
+    def test_extended(self):
+        a = [1, 2, 3, 4]
+        b = [1, 2, 3, 4, 5, 6, 7, 8]
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 2, 3, 4, 5, 6, 7, 8])
+
+    def test_extended_reversed(self):
+        a = [1, 2, 3, 4, 5, 6, 7, 8]
+        b = [1, 2, 3, 4]
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 2, 3, 4, 5, 6, 7, 8])
+
+    def test_prepended(self):
+        a = [5, 6, 7, 8]
+        b = [1, 2, 3, 4, 5, 6, 7, 8]
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 2, 3, 4, 5, 6, 7, 8])
+
+    def test_mixed(self):
+        a = [1, 2, 3, 4]
+        b = [5, 1, 2, 6, 4]
+        c = list_union(a, b)
+        self.assertEqual(c, [5, 1, 2, 3, 6, 4])
+
+    def test_mixed_reversed(self):
+        a = [5, 1, 2, 6, 4]
+        b = [1, 2, 3, 4]
+        c = list_union(a, b)
+        self.assertEqual(c, [5, 1, 2, 6, 3, 4])
+
+    def test_different_order(self):
+        a = [1, 2, 3, 4]
+        b = [2, 3, 1, 4]
+        c = list_union(a, b)
+        self.assertEqual(c, [2, 3, 1, 4])
+
+    def test_different_order2(self):
+        a = [1, 2, 3, 4, 5, 6]
+        b = [3, 1, 4, 2, 5, 6]
+        c = list_union(a, b)
+        self.assertEqual(c, [3, 1, 4, 2, 5, 6])
+
+    def test_different_order_superset(self):
+        a = [1, 2, 3, 4]
+        b = [4, 3, 1, 2, 5, 6]
+        c = list_union(a, b)
+        self.assertEqual(c, [4, 3, 1, 2, 5, 6])
+
+    def test_completely_disjoint(self):
+        a = [1, 2, 3, 4]
+        b = [5, 6, 7, 8]
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 5, 2, 6, 3, 7, 4, 8])
+
+    def test_different_suffix(self):
+        a = [1, 2, 3, 4]
+        b = [1, 2, 5, 6]
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 2, 3, 5, 4, 6])
+
+    def test_different_prefix(self):
+        a = [1, 2, 3, 4]
+        b = [5, 6, 3, 4]
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 5, 2, 6, 3, 4])
+
+    def test_one_empty(self):
+        a = [1, 2, 3, 4]
+        b = []
+        c = list_union(a, b)
+        self.assertEqual(c, [1, 2, 3, 4])
+
+    def test_both_empty(self):
+        a = []
+        b = []
+        c = list_union(a, b)
+        self.assertEqual(c, [])
+
+class TestBisectLists(unittest.TestCase):
+    def test_sorted(self):
+        a = [1, 5, 7, 9]
+        b = [3, 5, 6, 8]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [1, 3, 5, 7])
+
+        d = bisect_lists(c, b)
+        self.assertEqual(d, [1, 3, 5, 7])
+
+        e = bisect_lists(a, c)
+        self.assertEqual(e, [1, 3, 5, 7])
+
+    def test_extended(self):
+        a = [1, 2, 3, 4]
+        b = [1, 2, 3, 4, 5, 6, 7, 8]
+        c = bisect_lists(a, b)
+        self.assertEqual(a, [1, 2, 3, 4])
+        self.assertEqual(b, [1, 2, 3, 4, 5, 6, 7, 8])
+        self.assertEqual(c, [1, 2, 3, 4, 5, 6])
+
+        d = bisect_lists(c, b)
+        self.assertEqual(d, [1, 2, 3, 4, 5, 6, 7])
+
+    def test_extended_reversed(self):
+        a = [1, 2, 3, 4, 5, 6, 7, 8]
+        b = [1, 2, 3, 4]
+        c = bisect_lists(a, b)
+        self.assertEqual(a, [1, 2, 3, 4, 5, 6, 7, 8])
+        self.assertEqual(b, [1, 2, 3, 4])
+        self.assertEqual(c, [1, 2, 3, 4, 5, 6])
+
+    def test_prepended(self):
+        a = [5, 6, 7, 8]
+        b = [1, 2, 3, 4, 5, 6, 7, 8]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [1, 2, 5, 6, 7, 8])
+
+    def test_both_different(self):
+        a = [1, 2, 3, 4]
+        b = [1, 2, 5, 6]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [1, 2, 3, 5])
+
+    def test_small_difference(self):
+        a = [1, 2, 3, 4]
+        b = [1, 2, 3, 5]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [1, 2, 3, 4])
+
+    def test_small_difference_with_different_order(self):
+        a = [2, 3, 1, 4]
+        b = [1, 2, 3, 5]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [1, 2, 3, 5])
+
+    def test_one_empty(self):
+        a = []
+        b = [1, 2, 3, 4]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [1, 2])
+
+    def test_both_empty(self):
+        a = []
+        b = []
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [])
+
+    def test_one_None(self):
+        a = None
+        b = [1, 2, 3, 4]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [1, 2])
+
+    def test_short_and_None(self):
+        a = None
+        b = [1]
+        c = bisect_lists(a, b)
+        self.assertEqual(c, [])
+
+    def test_empty_and_None(self):
+        a = None
+        b = []
+        c = bisect_lists(a, b)
+        self.assertEqual(c, None)
+
+class TestBisect(unittest.TestCase):
+    def test___init__(self):
+        b = Bisect(None, None, None, None)
+        self.assertIsNotNone(b)
+
+    def test_run(self):
+        def test_cb(hello):
+            return len(hello.write()) <= 2**14
+
+        bad = HugeCipherList()
+        good = VeryCompatible()
+        self.assertGreater(len(bad(b'').write()), 2**14)
+        self.assertLess(len(good(b'').write()), 2**14)
+
+        bi = Bisect(good, bad, "localhost", test_cb)
+
+        a, b = bi.run()
+
+        self.assertEqual(len(a.write()), 2**14-1)
+        self.assertEqual(len(b.write()), 2**14+1)
+
+    def test_run_with_extensions(self):
+        def test_cb(hello):
+            if not hello.extensions:
+                return True
+            a = next((x for x in hello.extensions
+                      if isinstance(x, SignatureAlgorithmsExtension)), None)
+            return a is None
+        good = IE_8_Win_XP()
+        bad = VeryCompatible()
+
+        self.assertTrue(test_cb(good(b'localhost')))
+        self.assertFalse(test_cb(bad(b'localhost')))
+
+        bi = Bisect(good, bad, "localhost", test_cb)
+
+        a, b = bi.run()
+
+        ext = next((x for x in a.extensions
+                    if isinstance(x, SignatureAlgorithmsExtension)), None)
+        self.assertIsNone(ext)
+        ext = next((x for x in b.extensions
+                    if isinstance(x, SignatureAlgorithmsExtension)), None)
+        self.assertIsNotNone(ext)
+
+    def test_run_with_extension_size(self):
+        def test_cb(hello):
+            return len(hello.write()) <= 2**14
+
+        bad = extend_with_ext_to_size(VeryCompatible(), 2**16)
+        good = VeryCompatible()
+
+        bi = Bisect(good, bad, "localhost", test_cb)
+
+        a, b = bi.run()
+
+        self.assertEqual(len(a.write()), 2**14)
+        self.assertEqual(len(b.write()), 2**14+1)
+
+    def test_run_with_ext_and_ciphers(self):
+        def test_cb(hello):
+            return len(hello.write()) <= 2**14
+
+        bad = extend_with_ext_to_size(VeryCompatible(), 2**15)
+        bad = append_ciphers_to_size(bad, 2**16)
+        good = VeryCompatible()
+
+        bi = Bisect(good, bad, "localhost", test_cb)
+
+        a, b = bi.run()
+
+        self.assertEqual(len(a.write()), 2**14)
+        self.assertEqual(len(b.write()), 2**14+1)