Annotation of pkgsrc/lang/python27/patches/patch-Lib_test_test__urlparse.py, revision 1.2
1.2 ! gutterid 1: $NetBSD: patch-Lib_test_test__urlparse.py,v 1.1 2021/10/10 03:00:59 gutteridge Exp $
1.1 gutterid 2:
3: Fix CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
4: Via Fedora:
5: https://src.fedoraproject.org/rpms/python2.7/blob/rawhide/f/00359-CVE-2021-23336.patch
6:
1.2 ! gutterid 7: Fix CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
! 8: Via Fedora:
! 9: https://src.fedoraproject.org/rpms/python2.7/raw/40dd05e5d77dbfa81777c9f84b704bc2239bf710/f/00377-CVE-2022-0391.patch
! 10:
1.1 gutterid 11: --- Lib/test/test_urlparse.py.orig 2020-04-19 21:13:39.000000000 +0000
12: +++ Lib/test/test_urlparse.py
13: @@ -3,6 +3,12 @@ import sys
14: import unicodedata
15: import unittest
16: import urlparse
17: +from test.support import EnvironmentVarGuard
18: +from warnings import catch_warnings, filterwarnings
19: +import tempfile
20: +import contextlib
21: +import os.path
22: +import shutil
23:
24: RFC1808_BASE = "http://a/b/c/d;p?q#f"
25: RFC2396_BASE = "http://a/b/c/d;p?q"
26: @@ -24,16 +30,29 @@ parse_qsl_test_cases = [
27: ("&a=b", [('a', 'b')]),
28: ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]),
29: ("a=1&a=2", [('a', '1'), ('a', '2')]),
30: +]
31: +
32: +parse_qsl_test_cases_semicolon = [
33: (";", []),
34: (";;", []),
35: (";a=b", [('a', 'b')]),
36: ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]),
37: ("a=1;a=2", [('a', '1'), ('a', '2')]),
38: - (b";", []),
39: - (b";;", []),
40: - (b";a=b", [(b'a', b'b')]),
41: - (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
42: - (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
43: +]
44: +
45: +parse_qsl_test_cases_legacy = [
46: + ("a=1;a=2&a=3", [('a', '1'), ('a', '2'), ('a', '3')]),
47: + ("a=1;b=2&c=3", [('a', '1'), ('b', '2'), ('c', '3')]),
48: + ("a=1&b=2&c=3;", [('a', '1'), ('b', '2'), ('c', '3')]),
49: +]
50: +
51: +parse_qsl_test_cases_warn = [
52: + (";a=b", [(';a', 'b')]),
53: + ("a=a+b;b=b+c", [('a', 'a b;b=b c')]),
54: + (b";a=b", [(b';a', b'b')]),
55: + (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]),
56: + ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]),
57: + (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]),
58: ]
59:
60: parse_qs_test_cases = [
61: @@ -57,6 +76,9 @@ parse_qs_test_cases = [
62: (b"&a=b", {b'a': [b'b']}),
63: (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
64: (b"a=1&a=2", {b'a': [b'1', b'2']}),
65: +]
66: +
67: +parse_qs_test_cases_semicolon = [
68: (";", {}),
69: (";;", {}),
70: (";a=b", {'a': ['b']}),
71: @@ -69,6 +91,24 @@ parse_qs_test_cases = [
72: (b"a=1;a=2", {b'a': [b'1', b'2']}),
73: ]
74:
75: +parse_qs_test_cases_legacy = [
76: + ("a=1;a=2&a=3", {'a': ['1', '2', '3']}),
77: + ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
78: + ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
79: + (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}),
80: + (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
81: + (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
82: +]
83: +
84: +parse_qs_test_cases_warn = [
85: + (";a=b", {';a': ['b']}),
86: + ("a=a+b;b=b+c", {'a': ['a b;b=b c']}),
87: + (b";a=b", {b';a': [b'b']}),
88: + (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}),
89: + ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}),
90: + (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}),
91: +]
92: +
93: class UrlParseTestCase(unittest.TestCase):
94:
95: def checkRoundtrips(self, url, parsed, split):
96: @@ -141,6 +181,40 @@ class UrlParseTestCase(unittest.TestCase
97: self.assertEqual(result, expect_without_blanks,
98: "Error parsing %r" % orig)
99:
100: + def test_qs_default_warn(self):
101: + for orig, expect in parse_qs_test_cases_warn:
102: + with catch_warnings(record=True) as w:
103: + filterwarnings(action='always',
104: + category=urlparse._QueryStringSeparatorWarning)
105: + result = urlparse.parse_qs(orig, keep_blank_values=True)
106: + self.assertEqual(result, expect, "Error parsing %r" % orig)
107: + self.assertEqual(len(w), 1)
108: + self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
109: +
110: + def test_qsl_default_warn(self):
111: + for orig, expect in parse_qsl_test_cases_warn:
112: + with catch_warnings(record=True) as w:
113: + filterwarnings(action='always',
114: + category=urlparse._QueryStringSeparatorWarning)
115: + result = urlparse.parse_qsl(orig, keep_blank_values=True)
116: + self.assertEqual(result, expect, "Error parsing %r" % orig)
117: + self.assertEqual(len(w), 1)
118: + self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
119: +
120: + def test_default_qs_no_warnings(self):
121: + for orig, expect in parse_qs_test_cases:
122: + with catch_warnings(record=True) as w:
123: + result = urlparse.parse_qs(orig, keep_blank_values=True)
124: + self.assertEqual(result, expect, "Error parsing %r" % orig)
125: + self.assertEqual(len(w), 0)
126: +
127: + def test_default_qsl_no_warnings(self):
128: + for orig, expect in parse_qsl_test_cases:
129: + with catch_warnings(record=True) as w:
130: + result = urlparse.parse_qsl(orig, keep_blank_values=True)
131: + self.assertEqual(result, expect, "Error parsing %r" % orig)
132: + self.assertEqual(len(w), 0)
133: +
134: def test_roundtrips(self):
135: testcases = [
136: ('file:///tmp/junk.txt',
1.2 ! gutterid 137: @@ -544,6 +618,55 @@ class UrlParseTestCase(unittest.TestCase
! 138: self.assertEqual(p1.path, '863-1234')
! 139: self.assertEqual(p1.params, 'phone-context=+1-914-555')
! 140:
! 141: + def test_urlsplit_remove_unsafe_bytes(self):
! 142: + # Remove ASCII tabs and newlines from input, for http common case scenario.
! 143: + url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
! 144: + p = urlparse.urlsplit(url)
! 145: + self.assertEqual(p.scheme, "http")
! 146: + self.assertEqual(p.netloc, "www.python.org")
! 147: + self.assertEqual(p.path, "/javascript:alert('msg')/")
! 148: + self.assertEqual(p.query, "query=something")
! 149: + self.assertEqual(p.fragment, "fragment")
! 150: + self.assertEqual(p.username, None)
! 151: + self.assertEqual(p.password, None)
! 152: + self.assertEqual(p.hostname, "www.python.org")
! 153: + self.assertEqual(p.port, None)
! 154: + self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")
! 155: +
! 156: + # Remove ASCII tabs and newlines from input as bytes, for http common case scenario.
! 157: + url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
! 158: + p = urlparse.urlsplit(url)
! 159: + self.assertEqual(p.scheme, b"http")
! 160: + self.assertEqual(p.netloc, b"www.python.org")
! 161: + self.assertEqual(p.path, b"/javascript:alert('msg')/")
! 162: + self.assertEqual(p.query, b"query=something")
! 163: + self.assertEqual(p.fragment, b"fragment")
! 164: + self.assertEqual(p.username, None)
! 165: + self.assertEqual(p.password, None)
! 166: + self.assertEqual(p.hostname, b"www.python.org")
! 167: + self.assertEqual(p.port, None)
! 168: + self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")
! 169: +
! 170: + # any scheme
! 171: + url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
! 172: + p = urlparse.urlsplit(url)
! 173: + self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
! 174: +
! 175: + # Remove ASCII tabs and newlines from input as bytes, any scheme.
! 176: + url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
! 177: + p = urlparse.urlsplit(url)
! 178: + self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
! 179: +
! 180: + # Unsafe bytes is not returned from urlparse cache.
! 181: + # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme
! 182: + url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
! 183: + scheme = "htt\nps"
! 184: + for _ in range(2):
! 185: + p = urlparse.urlsplit(url, scheme=scheme)
! 186: + self.assertEqual(p.scheme, "https")
! 187: + self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
! 188: +
! 189: +
! 190:
! 191: def test_attributes_bad_port(self):
! 192: """Check handling of non-integer ports."""
! 193: @@ -626,6 +749,132 @@ class UrlParseTestCase(unittest.TestCase
1.1 gutterid 194: self.assertEqual(urlparse.urlparse("http://www.python.org:80"),
195: ('http','www.python.org:80','','','',''))
196:
197: + def test_parse_qs_separator_bytes(self):
198: + expected = {b'a': [b'1'], b'b': [b'2']}
199: +
200: + result = urlparse.parse_qs(b'a=1;b=2', separator=b';')
201: + self.assertEqual(result, expected)
202: + result = urlparse.parse_qs(b'a=1;b=2', separator=';')
203: + self.assertEqual(result, expected)
204: + result = urlparse.parse_qs('a=1;b=2', separator=';')
205: + self.assertEqual(result, {'a': ['1'], 'b': ['2']})
206: +
207: + @contextlib.contextmanager
208: + def _qsl_sep_config(self, sep):
209: + """Context for the given parse_qsl default separator configured in config file"""
210: + old_filename = urlparse._QS_SEPARATOR_CONFIG_FILENAME
211: + urlparse._default_qs_separator = None
212: + try:
213: + tmpdirname = tempfile.mkdtemp()
214: + filename = os.path.join(tmpdirname, 'conf.cfg')
215: + with open(filename, 'w') as file:
216: + file.write('[parse_qs]\n')
217: + file.write('PYTHON_URLLIB_QS_SEPARATOR = {}'.format(sep))
218: + urlparse._QS_SEPARATOR_CONFIG_FILENAME = filename
219: + yield
220: + finally:
221: + urlparse._QS_SEPARATOR_CONFIG_FILENAME = old_filename
222: + urlparse._default_qs_separator = None
223: + shutil.rmtree(tmpdirname)
224: +
225: + def test_parse_qs_separator_semicolon(self):
226: + for orig, expect in parse_qs_test_cases_semicolon:
227: + result = urlparse.parse_qs(orig, separator=';')
228: + self.assertEqual(result, expect, "Error parsing %r" % orig)
229: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
230: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
231: + result = urlparse.parse_qs(orig)
232: + self.assertEqual(result, expect, "Error parsing %r" % orig)
233: + self.assertEqual(len(w), 0)
234: + with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
235: + result = urlparse.parse_qs(orig)
236: + self.assertEqual(result, expect, "Error parsing %r" % orig)
237: + self.assertEqual(len(w), 0)
238: +
239: + def test_parse_qsl_separator_semicolon(self):
240: + for orig, expect in parse_qsl_test_cases_semicolon:
241: + result = urlparse.parse_qsl(orig, separator=';')
242: + self.assertEqual(result, expect, "Error parsing %r" % orig)
243: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
244: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
245: + result = urlparse.parse_qsl(orig)
246: + self.assertEqual(result, expect, "Error parsing %r" % orig)
247: + self.assertEqual(len(w), 0)
248: + with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
249: + result = urlparse.parse_qsl(orig)
250: + self.assertEqual(result, expect, "Error parsing %r" % orig)
251: + self.assertEqual(len(w), 0)
252: +
253: + def test_parse_qs_separator_legacy(self):
254: + for orig, expect in parse_qs_test_cases_legacy:
255: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
256: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
257: + result = urlparse.parse_qs(orig)
258: + self.assertEqual(result, expect, "Error parsing %r" % orig)
259: + self.assertEqual(len(w), 0)
260: + with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
261: + result = urlparse.parse_qs(orig)
262: + self.assertEqual(result, expect, "Error parsing %r" % orig)
263: + self.assertEqual(len(w), 0)
264: +
265: + def test_parse_qsl_separator_legacy(self):
266: + for orig, expect in parse_qsl_test_cases_legacy:
267: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
268: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
269: + result = urlparse.parse_qsl(orig)
270: + self.assertEqual(result, expect, "Error parsing %r" % orig)
271: + self.assertEqual(len(w), 0)
272: + with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
273: + result = urlparse.parse_qsl(orig)
274: + self.assertEqual(result, expect, "Error parsing %r" % orig)
275: + self.assertEqual(len(w), 0)
276: +
277: + def test_parse_qs_separator_bad_value_env_or_config(self):
278: + for bad_sep in '', 'abc', 'safe', '&;', 'SEP':
279: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
280: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep
281: + with self.assertRaises(ValueError):
282: + urlparse.parse_qsl('a=1;b=2')
283: + with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w:
284: + with self.assertRaises(ValueError):
285: + urlparse.parse_qsl('a=1;b=2')
286: +
287: + def test_parse_qs_separator_bad_value_arg(self):
288: + for bad_sep in True, {}, '':
289: + with self.assertRaises(ValueError):
290: + urlparse.parse_qsl('a=1;b=2', separator=bad_sep)
291: +
292: + def test_parse_qs_separator_num_fields(self):
293: + for qs, sep in (
294: + ('a&b&c', '&'),
295: + ('a;b;c', ';'),
296: + ('a&b;c', 'legacy'),
297: + ):
298: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
299: + if sep != 'legacy':
300: + with self.assertRaises(ValueError):
301: + urlparse.parse_qsl(qs, separator=sep, max_num_fields=2)
302: + if sep:
303: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep
304: + with self.assertRaises(ValueError):
305: + urlparse.parse_qsl(qs, max_num_fields=2)
306: +
307: + def test_parse_qs_separator_priority(self):
308: + # env variable trumps config file
309: + with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ:
310: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!'
311: + result = urlparse.parse_qs('a=1!b=2~c=3')
312: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
313: + # argument trumps config file
314: + with self._qsl_sep_config('~'):
315: + result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
316: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
317: + # argument trumps env variable
318: + with EnvironmentVarGuard() as environ:
319: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~'
320: + result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
321: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
322: +
323: def test_urlsplit_normalization(self):
324: # Certain characters should never occur in the netloc,
325: # including under normalization.
CVSweb <webmaster@jp.NetBSD.org>