From 1f26852b2928536624d1115133ca3182e620b177 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 4 Aug 2016 22:39:34 +0200 Subject: [PATCH] size intolerance checks since TLSv1.3 client hello's will include initial client key share, verify that the server is able to accept and process big client hello messages --- cscan.py | 77 +++++++++++ cscan/bisector.py | 172 +++++++++++++++++++++++ cscan_tests/test_bisector.py | 258 +++++++++++++++++++++++++++++++++++ 3 files changed, 507 insertions(+) create mode 100644 cscan/bisector.py create mode 100644 cscan_tests/test_bisector.py diff --git a/cscan.py b/cscan.py index fb5ab0e..f685ef0 100644 --- a/cscan.py +++ b/cscan.py @@ -20,6 +20,7 @@ from cscan.config import Xmas_tree, IE_6, IE_8_Win_XP, \ from cscan.modifiers import no_sni, set_hello_version, set_record_version, \ no_extensions, truncate_ciphers_to_size, append_ciphers_to_size, \ extend_with_ext_to_size, add_empty_ext +from cscan.bisector import Bisect def scan_with_config(host, port, conf, hostname, __sentry=None, __cache={}): @@ -291,6 +292,82 @@ def scan_TLS_intolerancies(host, port, hostname): conf.extensions and not conf.ssl2)) + #for name in ["Xmas tree", "Huge Cipher List", + # "Huge Cipher List (trunc c/16388)"]: + # intolerancies[name] = all(conf_iterator(lambda conf: + # conf.name == name)) + + def test_cb(client_hello): + ret = scan_with_config(host, port, lambda _:client_hello, hostname) + return simple_inspector(ret) + + # most size intolerancies lie between 16385 and 16389 so short-circuit to + # them if possible + good_conf = next((configs[name] for name, result in results.items() + if simple_inspector(result)), None) + + if good_conf: + size_c_16382 = simple_inspector(scan_with_config(host, port, + append_ciphers_to_size(copy.deepcopy(good_conf), 16382), hostname)) + size_c_16392 = simple_inspector(scan_with_config(host, port, + append_ciphers_to_size(copy.deepcopy(good_conf), 16392), hostname)) + + if size_c_16382 and not size_c_16392: + good = append_ciphers_to_size(copy.deepcopy(good_conf), 16382) + bad = append_ciphers_to_size(copy.deepcopy(good_conf), 16392) + elif not size_c_16382: + good = good_conf + bad = append_ciphers_to_size(copy.deepcopy(good_conf), 16382) + else: + bad = append_ciphers_to_size(copy.deepcopy(good_conf), 65536) + size_c_65536 = simple_inspector(scan_with_config(host, port, + bad, hostname)) + if not size_c_65536: + good = None + intolerancies["size c/65536"] = False + else: + good = append_ciphers_to_size(copy.deepcopy(good_conf), 16392) + + if good: + bisect = Bisect(good, bad, hostname, test_cb) + good_h, bad_h = bisect.run() + intolerancies["size c/{0}".format(len(bad_h.write()))] = True + intolerancies["size c/{0}".format(len(good_h.write()))] = False + + # test extension size intolerance, again, most lie between 16385 + # and 16389 so short-circuit if possible + good_conf = next((configs[name] for name, result in results.items() + if configs[name].extensions and + simple_inspector(result)), None) + + if good_conf: + size_e_16382 = simple_inspector(scan_with_config(host, port, + extend_with_ext_to_size(copy.deepcopy(good_conf), 16382), hostname)) + size_e_16392 = simple_inspector(scan_with_config(host, port, + extend_with_ext_to_size(copy.deepcopy(good_conf), 16392), hostname)) + + if size_e_16382 and not size_e_16392: + good = extend_with_ext_to_size(copy.deepcopy(good_conf), 16382) + bad = extend_with_ext_to_size(copy.deepcopy(good_conf), 16392) + elif not size_e_16382: + good = good_conf + bad = extend_with_ext_to_size(copy.deepcopy(good_conf), 16382) + else: + bad = extend_with_ext_to_size(copy.deepcopy(good_conf), 65536) + size_e_65536 = simple_inspector(scan_with_config(host, port, + bad, hostname)) + if not size_e_65536: + good = None + intolerancies["size e/65536"] = False + else: + good = extend_with_ext_to_size(copy.deepcopy(good_conf), 16392) + + if good: + bisect = Bisect(good, bad, hostname, test_cb) + good_h, bad_h = bisect.run() + intolerancies["size e/{0}".format(len(bad_h.write()))] = True + intolerancies["size e/{0}".format(len(good_h.write()))] = False + if json_out: print(json.dumps(intolerancies)) else: diff --git a/cscan/bisector.py b/cscan/bisector.py new file mode 100644 index 0000000..bf35e7f --- /dev/null +++ b/cscan/bisector.py @@ -0,0 +1,172 @@ +# Copyright (c) 2015 Hubert Kario +# Released under Mozilla Public License Version 2.0 + +"""Find an itolerance through bisecting Client Hello""" + +import copy +from tlslite.extensions import PaddingExtension + +def list_union(first, second): + """Return an union between two lists, preserving order""" + first_i = iter(first) + second_i = iter(second) + first_s = set(first) + second_s = set(second) + + ret = [] + first_el = next(first_i, None) + second_el = next(second_i, None) + while first_el is not None and second_el is not None: + if first_el != second_el: + if first_el in second_s and second_el in first_s: + # the second list is longer, so take from it + ret.append(second_el) + # no discard as we would have duplicates + second_el = next(second_i, None) + continue + if first_el not in second_s: + ret.append(first_el) + first_s.discard(first_el) + first_el = next(first_i, None) + if second_el not in first_s: + ret.append(second_el) + second_s.discard(second_el) + second_el = next(second_i, None) + else: + ret.append(first_el) + first_s.discard(first_el) + second_s.discard(first_el) + first_el = next(first_i, None) + second_el = next(second_i, None) + while first_el: + if first_el not in second_s: + ret.append(first_el) + first_el = next(first_i, None) + while second_el: + if second_el not in first_s: + ret.append(second_el) + second_el = next(second_i, None) + return ret + + +def bisect_lists(first, second): + """Return a list that is in the "middle" between the given ones""" + # handle None special cases + if first is None and second is None: + return None + if first is not None and second is None: + first, second = second, first + if first is None and second is not None: + if len(second) == 0: + return None + elif len(second) == 1: + return [] + else: + first = [] + # make the second lists always the longer one + if len(first) > len(second): + second, first = first, second + first_s = set(first) + second_s = set(second) + union = list_union(first, second) + symmetric_diff = first_s.symmetric_difference(second_s) + # preserve order for the difference + symmetric_diff = [x for x in union if x in symmetric_diff] + half_diff = set(symmetric_diff[:len(symmetric_diff)//2]) + intersection = first_s & second_s + + return [x for x in union if x in half_diff or x in intersection] + + +def bisect_padding_extension(first, second): + if first is None and second is None: + return None + if first is not None and second is None: + first, second = second, first + if first is None and second is not None: + if len(second.paddingData) == 0: + return None + elif len(second.paddingData) == 1: + return PaddingExtension() + else: + first = PaddingExtension() + return PaddingExtension().create((len(first.paddingData) + + len(second.paddingData)) // 2) + + +def bisect_extensions(first, second): + # handle padding extension + if first is None and second is None: + return None + if first is not None and second is None: + first, second = second, first + if first is None and second is not None: + if len(second) == 0: + return None + if len(second) == 1: + return [] + first = [] + f_ext = next((x for x in first if isinstance(x, PaddingExtension)), None) + s_ext = next((x for x in second if isinstance(x, PaddingExtension)), None) + + ext = bisect_padding_extension(f_ext, s_ext) + if ext is None: + # remove the extension + return [x for x in first if not isinstance(x, PaddingExtension)] + else: + if f_ext is None: + return first + [ext] + # replace extension + return [ext if isinstance(x, PaddingExtension) else x for x in first] + + +def bisect_hellos(first, second): + """Return a client hello that is in the "middle" of two other""" + ret = copy.copy(first) + + ret.client_version = ((first.client_version[0] + second.client_version[0]) + // 2, + (first.client_version[1] + second.client_version[1]) + // 2) + ret.cipher_suites = bisect_lists(first.cipher_suites, second.cipher_suites) + ret.extensions = bisect_lists(first.extensions, second.extensions) + ret.compression_methods = bisect_lists(first.compression_methods, + second.compression_methods) + if first.extensions == ret.extensions \ + or second.extensions == ret.extensions: + ret.extensions = bisect_extensions(first.extensions, + second.extensions) + return ret + +class Bisect(object): + """ + Perform a bisection between two Client Hello's to find intolerance + + Tries to find a cause for intolerance by using a bisection-like + algorithm + """ + + def __init__(self, good, bad, hostname, callback): + """Set the generators for good and bad hello's and callback to test""" + self.good = good + self.bad = bad + if hostname is not None: + self.hostname = bytearray(hostname, 'utf-8') + else: + self.hostname = None + self.callback = callback + + def run(self): + good_hello = self.good(self.hostname) + bad_hello = self.bad(self.hostname) + middle = bisect_hellos(good_hello, bad_hello) + + while good_hello != middle and \ + middle != bad_hello: + if self.callback(middle): + good_hello = middle + else: + bad_hello = middle + middle = bisect_hellos(good_hello, bad_hello) + + return (good_hello, bad_hello) diff --git a/cscan_tests/test_bisector.py b/cscan_tests/test_bisector.py new file mode 100644 index 0000000..71d0242 --- /dev/null +++ b/cscan_tests/test_bisector.py @@ -0,0 +1,258 @@ +# Copyright (c) 2015 Hubert Kario +# Released under Mozilla Public License Version 2.0 + +try: + import unittest2 as unittest +except ImportError: + import unittest + +from tlslite.extensions import SignatureAlgorithmsExtension, SNIExtension, \ + SupportedGroupsExtension, ECPointFormatsExtension, TLSExtension +from cscan.config import HugeCipherList, VeryCompatible, IE_8_Win_XP +from cscan.bisector import bisect_lists, list_union, Bisect +from cscan.modifiers import extend_with_ext_to_size, append_ciphers_to_size + +class TestListUnion(unittest.TestCase): + def test_identical(self): + a = [1, 2, 3, 4] + b = [1, 2, 3, 4] + c = list_union(a, b) + self.assertEqual(c, [1, 2, 3, 4]) + + def test_extended(self): + a = [1, 2, 3, 4] + b = [1, 2, 3, 4, 5, 6, 7, 8] + c = list_union(a, b) + self.assertEqual(c, [1, 2, 3, 4, 5, 6, 7, 8]) + + def test_extended_reversed(self): + a = [1, 2, 3, 4, 5, 6, 7, 8] + b = [1, 2, 3, 4] + c = list_union(a, b) + self.assertEqual(c, [1, 2, 3, 4, 5, 6, 7, 8]) + + def test_prepended(self): + a = [5, 6, 7, 8] + b = [1, 2, 3, 4, 5, 6, 7, 8] + c = list_union(a, b) + self.assertEqual(c, [1, 2, 3, 4, 5, 6, 7, 8]) + + def test_mixed(self): + a = [1, 2, 3, 4] + b = [5, 1, 2, 6, 4] + c = list_union(a, b) + self.assertEqual(c, [5, 1, 2, 3, 6, 4]) + + def test_mixed_reversed(self): + a = [5, 1, 2, 6, 4] + b = [1, 2, 3, 4] + c = list_union(a, b) + self.assertEqual(c, [5, 1, 2, 6, 3, 4]) + + def test_different_order(self): + a = [1, 2, 3, 4] + b = [2, 3, 1, 4] + c = list_union(a, b) + self.assertEqual(c, [2, 3, 1, 4]) + + def test_different_order2(self): + a = [1, 2, 3, 4, 5, 6] + b = [3, 1, 4, 2, 5, 6] + c = list_union(a, b) + self.assertEqual(c, [3, 1, 4, 2, 5, 6]) + + def test_different_order_superset(self): + a = [1, 2, 3, 4] + b = [4, 3, 1, 2, 5, 6] + c = list_union(a, b) + self.assertEqual(c, [4, 3, 1, 2, 5, 6]) + + def test_completely_disjoint(self): + a = [1, 2, 3, 4] + b = [5, 6, 7, 8] + c = list_union(a, b) + self.assertEqual(c, [1, 5, 2, 6, 3, 7, 4, 8]) + + def test_different_suffix(self): + a = [1, 2, 3, 4] + b = [1, 2, 5, 6] + c = list_union(a, b) + self.assertEqual(c, [1, 2, 3, 5, 4, 6]) + + def test_different_prefix(self): + a = [1, 2, 3, 4] + b = [5, 6, 3, 4] + c = list_union(a, b) + self.assertEqual(c, [1, 5, 2, 6, 3, 4]) + + def test_one_empty(self): + a = [1, 2, 3, 4] + b = [] + c = list_union(a, b) + self.assertEqual(c, [1, 2, 3, 4]) + + def test_both_empty(self): + a = [] + b = [] + c = list_union(a, b) + self.assertEqual(c, []) + +class TestBisectLists(unittest.TestCase): + def test_sorted(self): + a = [1, 5, 7, 9] + b = [3, 5, 6, 8] + c = bisect_lists(a, b) + self.assertEqual(c, [1, 3, 5, 7]) + + d = bisect_lists(c, b) + self.assertEqual(d, [1, 3, 5, 7]) + + e = bisect_lists(a, c) + self.assertEqual(e, [1, 3, 5, 7]) + + def test_extended(self): + a = [1, 2, 3, 4] + b = [1, 2, 3, 4, 5, 6, 7, 8] + c = bisect_lists(a, b) + self.assertEqual(a, [1, 2, 3, 4]) + self.assertEqual(b, [1, 2, 3, 4, 5, 6, 7, 8]) + self.assertEqual(c, [1, 2, 3, 4, 5, 6]) + + d = bisect_lists(c, b) + self.assertEqual(d, [1, 2, 3, 4, 5, 6, 7]) + + def test_extended_reversed(self): + a = [1, 2, 3, 4, 5, 6, 7, 8] + b = [1, 2, 3, 4] + c = bisect_lists(a, b) + self.assertEqual(a, [1, 2, 3, 4, 5, 6, 7, 8]) + self.assertEqual(b, [1, 2, 3, 4]) + self.assertEqual(c, [1, 2, 3, 4, 5, 6]) + + def test_prepended(self): + a = [5, 6, 7, 8] + b = [1, 2, 3, 4, 5, 6, 7, 8] + c = bisect_lists(a, b) + self.assertEqual(c, [1, 2, 5, 6, 7, 8]) + + def test_both_different(self): + a = [1, 2, 3, 4] + b = [1, 2, 5, 6] + c = bisect_lists(a, b) + self.assertEqual(c, [1, 2, 3, 5]) + + def test_small_difference(self): + a = [1, 2, 3, 4] + b = [1, 2, 3, 5] + c = bisect_lists(a, b) + self.assertEqual(c, [1, 2, 3, 4]) + + def test_small_difference_with_different_order(self): + a = [2, 3, 1, 4] + b = [1, 2, 3, 5] + c = bisect_lists(a, b) + self.assertEqual(c, [1, 2, 3, 5]) + + def test_one_empty(self): + a = [] + b = [1, 2, 3, 4] + c = bisect_lists(a, b) + self.assertEqual(c, [1, 2]) + + def test_both_empty(self): + a = [] + b = [] + c = bisect_lists(a, b) + self.assertEqual(c, []) + + def test_one_None(self): + a = None + b = [1, 2, 3, 4] + c = bisect_lists(a, b) + self.assertEqual(c, [1, 2]) + + def test_short_and_None(self): + a = None + b = [1] + c = bisect_lists(a, b) + self.assertEqual(c, []) + + def test_empty_and_None(self): + a = None + b = [] + c = bisect_lists(a, b) + self.assertEqual(c, None) + +class TestBisect(unittest.TestCase): + def test___init__(self): + b = Bisect(None, None, None, None) + self.assertIsNotNone(b) + + def test_run(self): + def test_cb(hello): + return len(hello.write()) <= 2**14 + + bad = HugeCipherList() + good = VeryCompatible() + self.assertGreater(len(bad(b'').write()), 2**14) + self.assertLess(len(good(b'').write()), 2**14) + + bi = Bisect(good, bad, "localhost", test_cb) + + a, b = bi.run() + + self.assertEqual(len(a.write()), 2**14-1) + self.assertEqual(len(b.write()), 2**14+1) + + def test_run_with_extensions(self): + def test_cb(hello): + if not hello.extensions: + return True + a = next((x for x in hello.extensions + if isinstance(x, SignatureAlgorithmsExtension)), None) + return a is None + good = IE_8_Win_XP() + bad = VeryCompatible() + + self.assertTrue(test_cb(good(b'localhost'))) + self.assertFalse(test_cb(bad(b'localhost'))) + + bi = Bisect(good, bad, "localhost", test_cb) + + a, b = bi.run() + + ext = next((x for x in a.extensions + if isinstance(x, SignatureAlgorithmsExtension)), None) + self.assertIsNone(ext) + ext = next((x for x in b.extensions + if isinstance(x, SignatureAlgorithmsExtension)), None) + self.assertIsNotNone(ext) + + def test_run_with_extension_size(self): + def test_cb(hello): + return len(hello.write()) <= 2**14 + + bad = extend_with_ext_to_size(VeryCompatible(), 2**16) + good = VeryCompatible() + + bi = Bisect(good, bad, "localhost", test_cb) + + a, b = bi.run() + + self.assertEqual(len(a.write()), 2**14) + self.assertEqual(len(b.write()), 2**14+1) + + def test_run_with_ext_and_ciphers(self): + def test_cb(hello): + return len(hello.write()) <= 2**14 + + bad = extend_with_ext_to_size(VeryCompatible(), 2**15) + bad = append_ciphers_to_size(bad, 2**16) + good = VeryCompatible() + + bi = Bisect(good, bad, "localhost", test_cb) + + a, b = bi.run() + + self.assertEqual(len(a.write()), 2**14) + self.assertEqual(len(b.write()), 2**14+1)