Annotation of pkgsrc/lang/python27/patches/patch-Lib_test_test__urlparse.py, revision 1.1
1.1 ! gutterid 1: $NetBSD$
! 2:
! 3: Fix CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
! 4: Via Fedora:
! 5: https://src.fedoraproject.org/rpms/python2.7/blob/rawhide/f/00359-CVE-2021-23336.patch
! 6:
! 7: --- Lib/test/test_urlparse.py.orig 2020-04-19 21:13:39.000000000 +0000
! 8: +++ Lib/test/test_urlparse.py
! 9: @@ -3,6 +3,12 @@ import sys
! 10: import unicodedata
! 11: import unittest
! 12: import urlparse
! 13: +from test.support import EnvironmentVarGuard
! 14: +from warnings import catch_warnings, filterwarnings
! 15: +import tempfile
! 16: +import contextlib
! 17: +import os.path
! 18: +import shutil
! 19:
! 20: RFC1808_BASE = "http://a/b/c/d;p?q#f"
! 21: RFC2396_BASE = "http://a/b/c/d;p?q"
! 22: @@ -24,16 +30,29 @@ parse_qsl_test_cases = [
! 23: ("&a=b", [('a', 'b')]),
! 24: ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]),
! 25: ("a=1&a=2", [('a', '1'), ('a', '2')]),
! 26: +]
! 27: +
! 28: +parse_qsl_test_cases_semicolon = [
! 29: (";", []),
! 30: (";;", []),
! 31: (";a=b", [('a', 'b')]),
! 32: ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]),
! 33: ("a=1;a=2", [('a', '1'), ('a', '2')]),
! 34: - (b";", []),
! 35: - (b";;", []),
! 36: - (b";a=b", [(b'a', b'b')]),
! 37: - (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
! 38: - (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
! 39: +]
! 40: +
! 41: +parse_qsl_test_cases_legacy = [
! 42: + ("a=1;a=2&a=3", [('a', '1'), ('a', '2'), ('a', '3')]),
! 43: + ("a=1;b=2&c=3", [('a', '1'), ('b', '2'), ('c', '3')]),
! 44: + ("a=1&b=2&c=3;", [('a', '1'), ('b', '2'), ('c', '3')]),
! 45: +]
! 46: +
! 47: +parse_qsl_test_cases_warn = [
! 48: + (";a=b", [(';a', 'b')]),
! 49: + ("a=a+b;b=b+c", [('a', 'a b;b=b c')]),
! 50: + (b";a=b", [(b';a', b'b')]),
! 51: + (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]),
! 52: + ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]),
! 53: + (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]),
! 54: ]
! 55:
! 56: parse_qs_test_cases = [
! 57: @@ -57,6 +76,9 @@ parse_qs_test_cases = [
! 58: (b"&a=b", {b'a': [b'b']}),
! 59: (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
! 60: (b"a=1&a=2", {b'a': [b'1', b'2']}),
! 61: +]
! 62: +
! 63: +parse_qs_test_cases_semicolon = [
! 64: (";", {}),
! 65: (";;", {}),
! 66: (";a=b", {'a': ['b']}),
! 67: @@ -69,6 +91,24 @@ parse_qs_test_cases = [
! 68: (b"a=1;a=2", {b'a': [b'1', b'2']}),
! 69: ]
! 70:
! 71: +parse_qs_test_cases_legacy = [
! 72: + ("a=1;a=2&a=3", {'a': ['1', '2', '3']}),
! 73: + ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
! 74: + ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
! 75: + (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}),
! 76: + (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
! 77: + (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
! 78: +]
! 79: +
! 80: +parse_qs_test_cases_warn = [
! 81: + (";a=b", {';a': ['b']}),
! 82: + ("a=a+b;b=b+c", {'a': ['a b;b=b c']}),
! 83: + (b";a=b", {b';a': [b'b']}),
! 84: + (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}),
! 85: + ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}),
! 86: + (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}),
! 87: +]
! 88: +
! 89: class UrlParseTestCase(unittest.TestCase):
! 90:
! 91: def checkRoundtrips(self, url, parsed, split):
! 92: @@ -141,6 +181,40 @@ class UrlParseTestCase(unittest.TestCase
! 93: self.assertEqual(result, expect_without_blanks,
! 94: "Error parsing %r" % orig)
! 95:
! 96: + def test_qs_default_warn(self):
! 97: + for orig, expect in parse_qs_test_cases_warn:
! 98: + with catch_warnings(record=True) as w:
! 99: + filterwarnings(action='always',
! 100: + category=urlparse._QueryStringSeparatorWarning)
! 101: + result = urlparse.parse_qs(orig, keep_blank_values=True)
! 102: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 103: + self.assertEqual(len(w), 1)
! 104: + self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
! 105: +
! 106: + def test_qsl_default_warn(self):
! 107: + for orig, expect in parse_qsl_test_cases_warn:
! 108: + with catch_warnings(record=True) as w:
! 109: + filterwarnings(action='always',
! 110: + category=urlparse._QueryStringSeparatorWarning)
! 111: + result = urlparse.parse_qsl(orig, keep_blank_values=True)
! 112: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 113: + self.assertEqual(len(w), 1)
! 114: + self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
! 115: +
! 116: + def test_default_qs_no_warnings(self):
! 117: + for orig, expect in parse_qs_test_cases:
! 118: + with catch_warnings(record=True) as w:
! 119: + result = urlparse.parse_qs(orig, keep_blank_values=True)
! 120: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 121: + self.assertEqual(len(w), 0)
! 122: +
! 123: + def test_default_qsl_no_warnings(self):
! 124: + for orig, expect in parse_qsl_test_cases:
! 125: + with catch_warnings(record=True) as w:
! 126: + result = urlparse.parse_qsl(orig, keep_blank_values=True)
! 127: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 128: + self.assertEqual(len(w), 0)
! 129: +
! 130: def test_roundtrips(self):
! 131: testcases = [
! 132: ('file:///tmp/junk.txt',
! 133: @@ -626,6 +700,132 @@ class UrlParseTestCase(unittest.TestCase
! 134: self.assertEqual(urlparse.urlparse("http://www.python.org:80"),
! 135: ('http','www.python.org:80','','','',''))
! 136:
! 137: + def test_parse_qs_separator_bytes(self):
! 138: + expected = {b'a': [b'1'], b'b': [b'2']}
! 139: +
! 140: + result = urlparse.parse_qs(b'a=1;b=2', separator=b';')
! 141: + self.assertEqual(result, expected)
! 142: + result = urlparse.parse_qs(b'a=1;b=2', separator=';')
! 143: + self.assertEqual(result, expected)
! 144: + result = urlparse.parse_qs('a=1;b=2', separator=';')
! 145: + self.assertEqual(result, {'a': ['1'], 'b': ['2']})
! 146: +
! 147: + @contextlib.contextmanager
! 148: + def _qsl_sep_config(self, sep):
! 149: + """Context for the given parse_qsl default separator configured in config file"""
! 150: + old_filename = urlparse._QS_SEPARATOR_CONFIG_FILENAME
! 151: + urlparse._default_qs_separator = None
! 152: + try:
! 153: + tmpdirname = tempfile.mkdtemp()
! 154: + filename = os.path.join(tmpdirname, 'conf.cfg')
! 155: + with open(filename, 'w') as file:
! 156: + file.write('[parse_qs]\n')
! 157: + file.write('PYTHON_URLLIB_QS_SEPARATOR = {}'.format(sep))
! 158: + urlparse._QS_SEPARATOR_CONFIG_FILENAME = filename
! 159: + yield
! 160: + finally:
! 161: + urlparse._QS_SEPARATOR_CONFIG_FILENAME = old_filename
! 162: + urlparse._default_qs_separator = None
! 163: + shutil.rmtree(tmpdirname)
! 164: +
! 165: + def test_parse_qs_separator_semicolon(self):
! 166: + for orig, expect in parse_qs_test_cases_semicolon:
! 167: + result = urlparse.parse_qs(orig, separator=';')
! 168: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 169: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
! 170: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
! 171: + result = urlparse.parse_qs(orig)
! 172: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 173: + self.assertEqual(len(w), 0)
! 174: + with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
! 175: + result = urlparse.parse_qs(orig)
! 176: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 177: + self.assertEqual(len(w), 0)
! 178: +
! 179: + def test_parse_qsl_separator_semicolon(self):
! 180: + for orig, expect in parse_qsl_test_cases_semicolon:
! 181: + result = urlparse.parse_qsl(orig, separator=';')
! 182: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 183: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
! 184: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
! 185: + result = urlparse.parse_qsl(orig)
! 186: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 187: + self.assertEqual(len(w), 0)
! 188: + with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
! 189: + result = urlparse.parse_qsl(orig)
! 190: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 191: + self.assertEqual(len(w), 0)
! 192: +
! 193: + def test_parse_qs_separator_legacy(self):
! 194: + for orig, expect in parse_qs_test_cases_legacy:
! 195: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
! 196: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
! 197: + result = urlparse.parse_qs(orig)
! 198: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 199: + self.assertEqual(len(w), 0)
! 200: + with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
! 201: + result = urlparse.parse_qs(orig)
! 202: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 203: + self.assertEqual(len(w), 0)
! 204: +
! 205: + def test_parse_qsl_separator_legacy(self):
! 206: + for orig, expect in parse_qsl_test_cases_legacy:
! 207: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
! 208: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
! 209: + result = urlparse.parse_qsl(orig)
! 210: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 211: + self.assertEqual(len(w), 0)
! 212: + with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
! 213: + result = urlparse.parse_qsl(orig)
! 214: + self.assertEqual(result, expect, "Error parsing %r" % orig)
! 215: + self.assertEqual(len(w), 0)
! 216: +
! 217: + def test_parse_qs_separator_bad_value_env_or_config(self):
! 218: + for bad_sep in '', 'abc', 'safe', '&;', 'SEP':
! 219: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
! 220: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep
! 221: + with self.assertRaises(ValueError):
! 222: + urlparse.parse_qsl('a=1;b=2')
! 223: + with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w:
! 224: + with self.assertRaises(ValueError):
! 225: + urlparse.parse_qsl('a=1;b=2')
! 226: +
! 227: + def test_parse_qs_separator_bad_value_arg(self):
! 228: + for bad_sep in True, {}, '':
! 229: + with self.assertRaises(ValueError):
! 230: + urlparse.parse_qsl('a=1;b=2', separator=bad_sep)
! 231: +
! 232: + def test_parse_qs_separator_num_fields(self):
! 233: + for qs, sep in (
! 234: + ('a&b&c', '&'),
! 235: + ('a;b;c', ';'),
! 236: + ('a&b;c', 'legacy'),
! 237: + ):
! 238: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
! 239: + if sep != 'legacy':
! 240: + with self.assertRaises(ValueError):
! 241: + urlparse.parse_qsl(qs, separator=sep, max_num_fields=2)
! 242: + if sep:
! 243: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep
! 244: + with self.assertRaises(ValueError):
! 245: + urlparse.parse_qsl(qs, max_num_fields=2)
! 246: +
! 247: + def test_parse_qs_separator_priority(self):
! 248: + # env variable trumps config file
! 249: + with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ:
! 250: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!'
! 251: + result = urlparse.parse_qs('a=1!b=2~c=3')
! 252: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
! 253: + # argument trumps config file
! 254: + with self._qsl_sep_config('~'):
! 255: + result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
! 256: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
! 257: + # argument trumps env variable
! 258: + with EnvironmentVarGuard() as environ:
! 259: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~'
! 260: + result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
! 261: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
! 262: +
! 263: def test_urlsplit_normalization(self):
! 264: # Certain characters should never occur in the netloc,
! 265: # including under normalization.
CVSweb <webmaster@jp.NetBSD.org>