Annotation of pkgsrc/lang/python27/patches/patch-Lib_test_test__urlparse.py, revision 1.2

1.2     ! gutterid    1: $NetBSD: patch-Lib_test_test__urlparse.py,v 1.1 2021/10/10 03:00:59 gutteridge Exp $
1.1       gutterid    2: 
                      3: Fix CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
                      4: Via Fedora:
                      5: https://src.fedoraproject.org/rpms/python2.7/blob/rawhide/f/00359-CVE-2021-23336.patch
                      6: 
1.2     ! gutterid    7: Fix CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
        !             8: Via Fedora:
        !             9: https://src.fedoraproject.org/rpms/python2.7/raw/40dd05e5d77dbfa81777c9f84b704bc2239bf710/f/00377-CVE-2022-0391.patch
        !            10: 
1.1       gutterid   11: --- Lib/test/test_urlparse.py.orig     2020-04-19 21:13:39.000000000 +0000
                     12: +++ Lib/test/test_urlparse.py
                     13: @@ -3,6 +3,12 @@ import sys
                     14:  import unicodedata
                     15:  import unittest
                     16:  import urlparse
                     17: +from test.support import EnvironmentVarGuard
                     18: +from warnings import catch_warnings, filterwarnings
                     19: +import tempfile
                     20: +import contextlib
                     21: +import os.path
                     22: +import shutil
                     23:  
                     24:  RFC1808_BASE = "http://a/b/c/d;p?q#f"
                     25:  RFC2396_BASE = "http://a/b/c/d;p?q"
                     26: @@ -24,16 +30,29 @@ parse_qsl_test_cases = [
                     27:      ("&a=b", [('a', 'b')]),
                     28:      ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]),
                     29:      ("a=1&a=2", [('a', '1'), ('a', '2')]),
                     30: +]
                     31: +
                     32: +parse_qsl_test_cases_semicolon = [
                     33:      (";", []),
                     34:      (";;", []),
                     35:      (";a=b", [('a', 'b')]),
                     36:      ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]),
                     37:      ("a=1;a=2", [('a', '1'), ('a', '2')]),
                     38: -    (b";", []),
                     39: -    (b";;", []),
                     40: -    (b";a=b", [(b'a', b'b')]),
                     41: -    (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
                     42: -    (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
                     43: +]
                     44: +
                     45: +parse_qsl_test_cases_legacy = [
                     46: +    ("a=1;a=2&a=3", [('a', '1'), ('a', '2'), ('a', '3')]),
                     47: +    ("a=1;b=2&c=3", [('a', '1'), ('b', '2'), ('c', '3')]),
                     48: +    ("a=1&b=2&c=3;", [('a', '1'), ('b', '2'), ('c', '3')]),
                     49: +]
                     50: +
                     51: +parse_qsl_test_cases_warn = [
                     52: +    (";a=b", [(';a', 'b')]),
                     53: +    ("a=a+b;b=b+c", [('a', 'a b;b=b c')]),
                     54: +    (b";a=b", [(b';a', b'b')]),
                     55: +    (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]),
                     56: +    ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]),
                     57: +    (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]),
                     58:  ]
                     59:  
                     60:  parse_qs_test_cases = [
                     61: @@ -57,6 +76,9 @@ parse_qs_test_cases = [
                     62:      (b"&a=b", {b'a': [b'b']}),
                     63:      (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
                     64:      (b"a=1&a=2", {b'a': [b'1', b'2']}),
                     65: +]
                     66: +
                     67: +parse_qs_test_cases_semicolon = [
                     68:      (";", {}),
                     69:      (";;", {}),
                     70:      (";a=b", {'a': ['b']}),
                     71: @@ -69,6 +91,24 @@ parse_qs_test_cases = [
                     72:      (b"a=1;a=2", {b'a': [b'1', b'2']}),
                     73:  ]
                     74:  
                     75: +parse_qs_test_cases_legacy = [
                     76: +    ("a=1;a=2&a=3", {'a': ['1', '2', '3']}),
                     77: +    ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
                     78: +    ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
                     79: +    (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}),
                     80: +    (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
                     81: +    (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
                     82: +]
                     83: +
                     84: +parse_qs_test_cases_warn = [
                     85: +    (";a=b", {';a': ['b']}),
                     86: +    ("a=a+b;b=b+c", {'a': ['a b;b=b c']}),
                     87: +    (b";a=b", {b';a': [b'b']}),
                     88: +    (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}),
                     89: +    ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}),
                     90: +    (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}),
                     91: +]
                     92: +
                     93:  class UrlParseTestCase(unittest.TestCase):
                     94:  
                     95:      def checkRoundtrips(self, url, parsed, split):
                     96: @@ -141,6 +181,40 @@ class UrlParseTestCase(unittest.TestCase
                     97:              self.assertEqual(result, expect_without_blanks,
                     98:                      "Error parsing %r" % orig)
                     99:  
                    100: +    def test_qs_default_warn(self):
                    101: +        for orig, expect in parse_qs_test_cases_warn:
                    102: +            with catch_warnings(record=True) as w:
                    103: +                filterwarnings(action='always',
                    104: +                                        category=urlparse._QueryStringSeparatorWarning)
                    105: +                result = urlparse.parse_qs(orig, keep_blank_values=True)
                    106: +                self.assertEqual(result, expect, "Error parsing %r" % orig)
                    107: +            self.assertEqual(len(w), 1)
                    108: +            self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
                    109: +
                    110: +    def test_qsl_default_warn(self):
                    111: +        for orig, expect in parse_qsl_test_cases_warn:
                    112: +            with catch_warnings(record=True) as w:
                    113: +                filterwarnings(action='always',
                    114: +                               category=urlparse._QueryStringSeparatorWarning)
                    115: +                result = urlparse.parse_qsl(orig, keep_blank_values=True)
                    116: +                self.assertEqual(result, expect, "Error parsing %r" % orig)
                    117: +            self.assertEqual(len(w), 1)
                    118: +            self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
                    119: +
                    120: +    def test_default_qs_no_warnings(self):
                    121: +        for orig, expect in parse_qs_test_cases:
                    122: +            with catch_warnings(record=True) as w:
                    123: +                result = urlparse.parse_qs(orig, keep_blank_values=True)
                    124: +                self.assertEqual(result, expect, "Error parsing %r" % orig)
                    125: +            self.assertEqual(len(w), 0)
                    126: +
                    127: +    def test_default_qsl_no_warnings(self):
                    128: +        for orig, expect in parse_qsl_test_cases:
                    129: +            with catch_warnings(record=True) as w:
                    130: +                result = urlparse.parse_qsl(orig, keep_blank_values=True)
                    131: +                self.assertEqual(result, expect, "Error parsing %r" % orig)
                    132: +            self.assertEqual(len(w), 0)
                    133: +
                    134:      def test_roundtrips(self):
                    135:          testcases = [
                    136:              ('file:///tmp/junk.txt',
1.2     ! gutterid  137: @@ -544,6 +618,55 @@ class UrlParseTestCase(unittest.TestCase
        !           138:          self.assertEqual(p1.path, '863-1234')
        !           139:          self.assertEqual(p1.params, 'phone-context=+1-914-555')
        !           140:  
        !           141: +    def test_urlsplit_remove_unsafe_bytes(self):
        !           142: +        # Remove ASCII tabs and newlines from input, for http common case scenario.
        !           143: +        url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
        !           144: +        p = urlparse.urlsplit(url)
        !           145: +        self.assertEqual(p.scheme, "http")
        !           146: +        self.assertEqual(p.netloc, "www.python.org")
        !           147: +        self.assertEqual(p.path, "/javascript:alert('msg')/")
        !           148: +        self.assertEqual(p.query, "query=something")
        !           149: +        self.assertEqual(p.fragment, "fragment")
        !           150: +        self.assertEqual(p.username, None)
        !           151: +        self.assertEqual(p.password, None)
        !           152: +        self.assertEqual(p.hostname, "www.python.org")
        !           153: +        self.assertEqual(p.port, None)
        !           154: +        self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")
        !           155: +
        !           156: +        # Remove ASCII tabs and newlines from input as bytes, for http common case scenario.
        !           157: +        url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
        !           158: +        p = urlparse.urlsplit(url)
        !           159: +        self.assertEqual(p.scheme, b"http")
        !           160: +        self.assertEqual(p.netloc, b"www.python.org")
        !           161: +        self.assertEqual(p.path, b"/javascript:alert('msg')/")
        !           162: +        self.assertEqual(p.query, b"query=something")
        !           163: +        self.assertEqual(p.fragment, b"fragment")
        !           164: +        self.assertEqual(p.username, None)
        !           165: +        self.assertEqual(p.password, None)
        !           166: +        self.assertEqual(p.hostname, b"www.python.org")
        !           167: +        self.assertEqual(p.port, None)
        !           168: +        self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")
        !           169: +
        !           170: +        # any scheme
        !           171: +        url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
        !           172: +        p = urlparse.urlsplit(url)
        !           173: +        self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
        !           174: +
        !           175: +        # Remove ASCII tabs and newlines from input as bytes, any scheme.
        !           176: +        url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
        !           177: +        p = urlparse.urlsplit(url)
        !           178: +        self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
        !           179: +
        !           180: +        # Unsafe bytes is not returned from urlparse cache.
        !           181: +        # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme
        !           182: +        url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
        !           183: +        scheme = "htt\nps"
        !           184: +        for _ in range(2):
        !           185: +            p = urlparse.urlsplit(url, scheme=scheme)
        !           186: +            self.assertEqual(p.scheme, "https")
        !           187: +            self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
        !           188: +
        !           189: +
        !           190:  
        !           191:      def test_attributes_bad_port(self):
        !           192:          """Check handling of non-integer ports."""
        !           193: @@ -626,6 +749,132 @@ class UrlParseTestCase(unittest.TestCase
1.1       gutterid  194:          self.assertEqual(urlparse.urlparse("http://www.python.org:80"),
                    195:                  ('http','www.python.org:80','','','',''))
                    196:  
                    197: +    def test_parse_qs_separator_bytes(self):
                    198: +        expected = {b'a': [b'1'], b'b': [b'2']}
                    199: +
                    200: +        result = urlparse.parse_qs(b'a=1;b=2', separator=b';')
                    201: +        self.assertEqual(result, expected)
                    202: +        result = urlparse.parse_qs(b'a=1;b=2', separator=';')
                    203: +        self.assertEqual(result, expected)
                    204: +        result = urlparse.parse_qs('a=1;b=2', separator=';')
                    205: +        self.assertEqual(result, {'a': ['1'], 'b': ['2']})
                    206: +
                    207: +    @contextlib.contextmanager
                    208: +    def _qsl_sep_config(self, sep):
                    209: +        """Context for the given parse_qsl default separator configured in config file"""
                    210: +        old_filename = urlparse._QS_SEPARATOR_CONFIG_FILENAME
                    211: +        urlparse._default_qs_separator = None
                    212: +        try:
                    213: +            tmpdirname = tempfile.mkdtemp()
                    214: +            filename = os.path.join(tmpdirname, 'conf.cfg')
                    215: +            with open(filename, 'w') as file:
                    216: +                file.write('[parse_qs]\n')
                    217: +                file.write('PYTHON_URLLIB_QS_SEPARATOR = {}'.format(sep))
                    218: +            urlparse._QS_SEPARATOR_CONFIG_FILENAME = filename
                    219: +            yield
                    220: +        finally:
                    221: +            urlparse._QS_SEPARATOR_CONFIG_FILENAME = old_filename
                    222: +            urlparse._default_qs_separator = None
                    223: +            shutil.rmtree(tmpdirname)
                    224: +
                    225: +    def test_parse_qs_separator_semicolon(self):
                    226: +        for orig, expect in parse_qs_test_cases_semicolon:
                    227: +            result = urlparse.parse_qs(orig, separator=';')
                    228: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    229: +            with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
                    230: +                environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
                    231: +                result = urlparse.parse_qs(orig)
                    232: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    233: +            self.assertEqual(len(w), 0)
                    234: +            with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
                    235: +                result = urlparse.parse_qs(orig)
                    236: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    237: +            self.assertEqual(len(w), 0)
                    238: +
                    239: +    def test_parse_qsl_separator_semicolon(self):
                    240: +        for orig, expect in parse_qsl_test_cases_semicolon:
                    241: +            result = urlparse.parse_qsl(orig, separator=';')
                    242: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    243: +            with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
                    244: +                environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
                    245: +                result = urlparse.parse_qsl(orig)
                    246: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    247: +            self.assertEqual(len(w), 0)
                    248: +            with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
                    249: +                result = urlparse.parse_qsl(orig)
                    250: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    251: +            self.assertEqual(len(w), 0)
                    252: +
                    253: +    def test_parse_qs_separator_legacy(self):
                    254: +        for orig, expect in parse_qs_test_cases_legacy:
                    255: +            with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
                    256: +                environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
                    257: +                result = urlparse.parse_qs(orig)
                    258: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    259: +            self.assertEqual(len(w), 0)
                    260: +            with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
                    261: +                result = urlparse.parse_qs(orig)
                    262: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    263: +            self.assertEqual(len(w), 0)
                    264: +
                    265: +    def test_parse_qsl_separator_legacy(self):
                    266: +        for orig, expect in parse_qsl_test_cases_legacy:
                    267: +            with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
                    268: +                environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
                    269: +                result = urlparse.parse_qsl(orig)
                    270: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    271: +            self.assertEqual(len(w), 0)
                    272: +            with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
                    273: +                result = urlparse.parse_qsl(orig)
                    274: +            self.assertEqual(result, expect, "Error parsing %r" % orig)
                    275: +            self.assertEqual(len(w), 0)
                    276: +
                    277: +    def test_parse_qs_separator_bad_value_env_or_config(self):
                    278: +        for bad_sep in '', 'abc', 'safe', '&;', 'SEP':
                    279: +            with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
                    280: +                environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep
                    281: +                with self.assertRaises(ValueError):
                    282: +                    urlparse.parse_qsl('a=1;b=2')
                    283: +            with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w:
                    284: +                with self.assertRaises(ValueError):
                    285: +                    urlparse.parse_qsl('a=1;b=2')
                    286: +
                    287: +    def test_parse_qs_separator_bad_value_arg(self):
                    288: +        for bad_sep in True, {}, '':
                    289: +            with self.assertRaises(ValueError):
                    290: +                urlparse.parse_qsl('a=1;b=2', separator=bad_sep)
                    291: +
                    292: +    def test_parse_qs_separator_num_fields(self):
                    293: +        for qs, sep in (
                    294: +            ('a&b&c', '&'),
                    295: +            ('a;b;c', ';'),
                    296: +            ('a&b;c', 'legacy'),
                    297: +        ):
                    298: +            with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
                    299: +                if sep != 'legacy':
                    300: +                    with self.assertRaises(ValueError):
                    301: +                        urlparse.parse_qsl(qs, separator=sep, max_num_fields=2)
                    302: +                if sep:
                    303: +                    environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep
                    304: +                with self.assertRaises(ValueError):
                    305: +                    urlparse.parse_qsl(qs, max_num_fields=2)
                    306: +
                    307: +    def test_parse_qs_separator_priority(self):
                    308: +        # env variable trumps config file
                    309: +        with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ:
                    310: +            environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!'
                    311: +            result = urlparse.parse_qs('a=1!b=2~c=3')
                    312: +            self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
                    313: +        # argument trumps config file
                    314: +        with self._qsl_sep_config('~'):
                    315: +            result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
                    316: +            self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
                    317: +        # argument trumps env variable
                    318: +        with EnvironmentVarGuard() as environ:
                    319: +            environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~'
                    320: +            result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
                    321: +            self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
                    322: +
                    323:      def test_urlsplit_normalization(self):
                    324:          # Certain characters should never occur in the netloc,
                    325:          # including under normalization.

CVSweb <webmaster@jp.NetBSD.org>