Annotation of pkgsrc/lang/python27/patches/patch-Lib_test_test__urlparse.py, revision 1.3
1.3 ! gutterid 1: $NetBSD: patch-Lib_test_test__urlparse.py,v 1.2 2022/02/25 22:41:32 gutteridge Exp $
1.1 gutterid 2:
3: Fix CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
4: Via Fedora:
5: https://src.fedoraproject.org/rpms/python2.7/blob/rawhide/f/00359-CVE-2021-23336.patch
6:
1.2 gutterid 7: Fix CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
8: Via Fedora:
9: https://src.fedoraproject.org/rpms/python2.7/raw/40dd05e5d77dbfa81777c9f84b704bc2239bf710/f/00377-CVE-2022-0391.patch
10:
1.3 ! gutterid 11: Fix CVE-2023-24329: Add more sanitizing to respect the "Remove any leading C0 control or space from input" rule
! 12: Via Fedora:
! 13: https://src.fedoraproject.org/rpms/python2.7/c/3f00cdccd59ef2955a7f4b4c42bb59c631cce4c1.patch
! 14:
1.1 gutterid 15: --- Lib/test/test_urlparse.py.orig 2020-04-19 21:13:39.000000000 +0000
16: +++ Lib/test/test_urlparse.py
17: @@ -3,6 +3,12 @@ import sys
18: import unicodedata
19: import unittest
20: import urlparse
21: +from test.support import EnvironmentVarGuard
22: +from warnings import catch_warnings, filterwarnings
23: +import tempfile
24: +import contextlib
25: +import os.path
26: +import shutil
27:
28: RFC1808_BASE = "http://a/b/c/d;p?q#f"
29: RFC2396_BASE = "http://a/b/c/d;p?q"
30: @@ -24,16 +30,29 @@ parse_qsl_test_cases = [
31: ("&a=b", [('a', 'b')]),
32: ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]),
33: ("a=1&a=2", [('a', '1'), ('a', '2')]),
34: +]
35: +
36: +parse_qsl_test_cases_semicolon = [
37: (";", []),
38: (";;", []),
39: (";a=b", [('a', 'b')]),
40: ("a=a+b;b=b+c", [('a', 'a b'), ('b', 'b c')]),
41: ("a=1;a=2", [('a', '1'), ('a', '2')]),
42: - (b";", []),
43: - (b";;", []),
44: - (b";a=b", [(b'a', b'b')]),
45: - (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
46: - (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]),
47: +]
48: +
49: +parse_qsl_test_cases_legacy = [
50: + ("a=1;a=2&a=3", [('a', '1'), ('a', '2'), ('a', '3')]),
51: + ("a=1;b=2&c=3", [('a', '1'), ('b', '2'), ('c', '3')]),
52: + ("a=1&b=2&c=3;", [('a', '1'), ('b', '2'), ('c', '3')]),
53: +]
54: +
55: +parse_qsl_test_cases_warn = [
56: + (";a=b", [(';a', 'b')]),
57: + ("a=a+b;b=b+c", [('a', 'a b;b=b c')]),
58: + (b";a=b", [(b';a', b'b')]),
59: + (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]),
60: + ("a=1;a=2&a=3", [('a', '1;a=2'), ('a', '3')]),
61: + (b"a=1;a=2&a=3", [(b'a', b'1;a=2'), (b'a', b'3')]),
62: ]
63:
64: parse_qs_test_cases = [
65: @@ -57,6 +76,9 @@ parse_qs_test_cases = [
66: (b"&a=b", {b'a': [b'b']}),
67: (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}),
68: (b"a=1&a=2", {b'a': [b'1', b'2']}),
69: +]
70: +
71: +parse_qs_test_cases_semicolon = [
72: (";", {}),
73: (";;", {}),
74: (";a=b", {'a': ['b']}),
75: @@ -69,6 +91,24 @@ parse_qs_test_cases = [
76: (b"a=1;a=2", {b'a': [b'1', b'2']}),
77: ]
78:
79: +parse_qs_test_cases_legacy = [
80: + ("a=1;a=2&a=3", {'a': ['1', '2', '3']}),
81: + ("a=1;b=2&c=3", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
82: + ("a=1&b=2&c=3;", {'a': ['1'], 'b': ['2'], 'c': ['3']}),
83: + (b"a=1;a=2&a=3", {b'a': [b'1', b'2', b'3']}),
84: + (b"a=1;b=2&c=3", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
85: + (b"a=1&b=2&c=3;", {b'a': [b'1'], b'b': [b'2'], b'c': [b'3']}),
86: +]
87: +
88: +parse_qs_test_cases_warn = [
89: + (";a=b", {';a': ['b']}),
90: + ("a=a+b;b=b+c", {'a': ['a b;b=b c']}),
91: + (b";a=b", {b';a': [b'b']}),
92: + (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}),
93: + ("a=1;a=2&a=3", {'a': ['1;a=2', '3']}),
94: + (b"a=1;a=2&a=3", {b'a': [b'1;a=2', b'3']}),
95: +]
96: +
97: class UrlParseTestCase(unittest.TestCase):
98:
99: def checkRoundtrips(self, url, parsed, split):
100: @@ -141,6 +181,40 @@ class UrlParseTestCase(unittest.TestCase
101: self.assertEqual(result, expect_without_blanks,
102: "Error parsing %r" % orig)
103:
104: + def test_qs_default_warn(self):
105: + for orig, expect in parse_qs_test_cases_warn:
106: + with catch_warnings(record=True) as w:
107: + filterwarnings(action='always',
108: + category=urlparse._QueryStringSeparatorWarning)
109: + result = urlparse.parse_qs(orig, keep_blank_values=True)
110: + self.assertEqual(result, expect, "Error parsing %r" % orig)
111: + self.assertEqual(len(w), 1)
112: + self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
113: +
114: + def test_qsl_default_warn(self):
115: + for orig, expect in parse_qsl_test_cases_warn:
116: + with catch_warnings(record=True) as w:
117: + filterwarnings(action='always',
118: + category=urlparse._QueryStringSeparatorWarning)
119: + result = urlparse.parse_qsl(orig, keep_blank_values=True)
120: + self.assertEqual(result, expect, "Error parsing %r" % orig)
121: + self.assertEqual(len(w), 1)
122: + self.assertEqual(w[0].category, urlparse._QueryStringSeparatorWarning)
123: +
124: + def test_default_qs_no_warnings(self):
125: + for orig, expect in parse_qs_test_cases:
126: + with catch_warnings(record=True) as w:
127: + result = urlparse.parse_qs(orig, keep_blank_values=True)
128: + self.assertEqual(result, expect, "Error parsing %r" % orig)
129: + self.assertEqual(len(w), 0)
130: +
131: + def test_default_qsl_no_warnings(self):
132: + for orig, expect in parse_qsl_test_cases:
133: + with catch_warnings(record=True) as w:
134: + result = urlparse.parse_qsl(orig, keep_blank_values=True)
135: + self.assertEqual(result, expect, "Error parsing %r" % orig)
136: + self.assertEqual(len(w), 0)
137: +
138: def test_roundtrips(self):
139: testcases = [
140: ('file:///tmp/junk.txt',
1.3 ! gutterid 141: @@ -544,6 +618,112 @@ class UrlParseTestCase(unittest.TestCase
1.2 gutterid 142: self.assertEqual(p1.path, '863-1234')
143: self.assertEqual(p1.params, 'phone-context=+1-914-555')
144:
145: + def test_urlsplit_remove_unsafe_bytes(self):
146: + # Remove ASCII tabs and newlines from input, for http common case scenario.
147: + url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
148: + p = urlparse.urlsplit(url)
149: + self.assertEqual(p.scheme, "http")
150: + self.assertEqual(p.netloc, "www.python.org")
151: + self.assertEqual(p.path, "/javascript:alert('msg')/")
152: + self.assertEqual(p.query, "query=something")
153: + self.assertEqual(p.fragment, "fragment")
154: + self.assertEqual(p.username, None)
155: + self.assertEqual(p.password, None)
156: + self.assertEqual(p.hostname, "www.python.org")
157: + self.assertEqual(p.port, None)
158: + self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")
159: +
160: + # Remove ASCII tabs and newlines from input as bytes, for http common case scenario.
161: + url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
162: + p = urlparse.urlsplit(url)
163: + self.assertEqual(p.scheme, b"http")
164: + self.assertEqual(p.netloc, b"www.python.org")
165: + self.assertEqual(p.path, b"/javascript:alert('msg')/")
166: + self.assertEqual(p.query, b"query=something")
167: + self.assertEqual(p.fragment, b"fragment")
168: + self.assertEqual(p.username, None)
169: + self.assertEqual(p.password, None)
170: + self.assertEqual(p.hostname, b"www.python.org")
171: + self.assertEqual(p.port, None)
172: + self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")
173: +
174: + # any scheme
175: + url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
176: + p = urlparse.urlsplit(url)
177: + self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
178: +
179: + # Remove ASCII tabs and newlines from input as bytes, any scheme.
180: + url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
181: + p = urlparse.urlsplit(url)
182: + self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
183: +
184: + # Unsafe bytes is not returned from urlparse cache.
185: + # scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme
186: + url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
187: + scheme = "htt\nps"
188: + for _ in range(2):
189: + p = urlparse.urlsplit(url, scheme=scheme)
190: + self.assertEqual(p.scheme, "https")
191: + self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
192: +
1.3 ! gutterid 193: + def test_urlsplit_strip_url(self):
! 194: + noise = "".join([chr(i) for i in range(0, 0x20 + 1)])
! 195: + base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
1.2 gutterid 196: +
1.3 ! gutterid 197: + url = noise.decode("utf-8") + base_url
! 198: + p = urlparse.urlsplit(url)
! 199: + self.assertEqual(p.scheme, "http")
! 200: + self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
! 201: + self.assertEqual(p.path, "/doc/")
! 202: + self.assertEqual(p.query, "query=yes")
! 203: + self.assertEqual(p.fragment, "frag")
! 204: + self.assertEqual(p.username, "User")
! 205: + self.assertEqual(p.password, "Pass")
! 206: + self.assertEqual(p.hostname, "www.python.org")
! 207: + self.assertEqual(p.port, 80)
! 208: + self.assertEqual(p.geturl(), base_url)
! 209: +
! 210: + url = noise + base_url.encode("utf-8")
! 211: + p = urlparse.urlsplit(url)
! 212: + self.assertEqual(p.scheme, b"http")
! 213: + self.assertEqual(p.netloc, b"User:Pass@www.python.org:080")
! 214: + self.assertEqual(p.path, b"/doc/")
! 215: + self.assertEqual(p.query, b"query=yes")
! 216: + self.assertEqual(p.fragment, b"frag")
! 217: + self.assertEqual(p.username, b"User")
! 218: + self.assertEqual(p.password, b"Pass")
! 219: + self.assertEqual(p.hostname, b"www.python.org")
! 220: + self.assertEqual(p.port, 80)
! 221: + self.assertEqual(p.geturl(), base_url.encode("utf-8"))
! 222: +
! 223: + # Test that trailing space is preserved as some applications rely on
! 224: + # this within query strings.
! 225: + query_spaces_url = "https://www.python.org:88/doc/?query= "
! 226: + p = urlparse.urlsplit(noise.decode("utf-8") + query_spaces_url)
! 227: + self.assertEqual(p.scheme, "https")
! 228: + self.assertEqual(p.netloc, "www.python.org:88")
! 229: + self.assertEqual(p.path, "/doc/")
! 230: + self.assertEqual(p.query, "query= ")
! 231: + self.assertEqual(p.port, 88)
! 232: + self.assertEqual(p.geturl(), query_spaces_url)
! 233: +
! 234: + p = urlparse.urlsplit("www.pypi.org ")
! 235: + # That "hostname" gets considered a "path" due to the
! 236: + # trailing space and our existing logic... YUCK...
! 237: + # and re-assembles via geturl aka unurlsplit into the original.
! 238: + # django.core.validators.URLValidator (at least through v3.2) relies on
! 239: + # this, for better or worse, to catch it in a ValidationError via its
! 240: + # regular expressions.
! 241: + # Here we test the basic round trip concept of such a trailing space.
! 242: + self.assertEqual(urlparse.urlunsplit(p), "www.pypi.org ")
! 243: +
! 244: + # with scheme as cache-key
! 245: + url = "//www.python.org/"
! 246: + scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8")
! 247: + for _ in range(2):
! 248: + p = urlparse.urlsplit(url, scheme=scheme)
! 249: + self.assertEqual(p.scheme, "https")
! 250: + self.assertEqual(p.geturl(), "https://www.python.org/")
1.2 gutterid 251:
252: def test_attributes_bad_port(self):
253: """Check handling of non-integer ports."""
1.3 ! gutterid 254: @@ -626,6 +806,132 @@ class UrlParseTestCase(unittest.TestCase
1.1 gutterid 255: self.assertEqual(urlparse.urlparse("http://www.python.org:80"),
256: ('http','www.python.org:80','','','',''))
257:
258: + def test_parse_qs_separator_bytes(self):
259: + expected = {b'a': [b'1'], b'b': [b'2']}
260: +
261: + result = urlparse.parse_qs(b'a=1;b=2', separator=b';')
262: + self.assertEqual(result, expected)
263: + result = urlparse.parse_qs(b'a=1;b=2', separator=';')
264: + self.assertEqual(result, expected)
265: + result = urlparse.parse_qs('a=1;b=2', separator=';')
266: + self.assertEqual(result, {'a': ['1'], 'b': ['2']})
267: +
268: + @contextlib.contextmanager
269: + def _qsl_sep_config(self, sep):
270: + """Context for the given parse_qsl default separator configured in config file"""
271: + old_filename = urlparse._QS_SEPARATOR_CONFIG_FILENAME
272: + urlparse._default_qs_separator = None
273: + try:
274: + tmpdirname = tempfile.mkdtemp()
275: + filename = os.path.join(tmpdirname, 'conf.cfg')
276: + with open(filename, 'w') as file:
277: + file.write('[parse_qs]\n')
278: + file.write('PYTHON_URLLIB_QS_SEPARATOR = {}'.format(sep))
279: + urlparse._QS_SEPARATOR_CONFIG_FILENAME = filename
280: + yield
281: + finally:
282: + urlparse._QS_SEPARATOR_CONFIG_FILENAME = old_filename
283: + urlparse._default_qs_separator = None
284: + shutil.rmtree(tmpdirname)
285: +
286: + def test_parse_qs_separator_semicolon(self):
287: + for orig, expect in parse_qs_test_cases_semicolon:
288: + result = urlparse.parse_qs(orig, separator=';')
289: + self.assertEqual(result, expect, "Error parsing %r" % orig)
290: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
291: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
292: + result = urlparse.parse_qs(orig)
293: + self.assertEqual(result, expect, "Error parsing %r" % orig)
294: + self.assertEqual(len(w), 0)
295: + with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
296: + result = urlparse.parse_qs(orig)
297: + self.assertEqual(result, expect, "Error parsing %r" % orig)
298: + self.assertEqual(len(w), 0)
299: +
300: + def test_parse_qsl_separator_semicolon(self):
301: + for orig, expect in parse_qsl_test_cases_semicolon:
302: + result = urlparse.parse_qsl(orig, separator=';')
303: + self.assertEqual(result, expect, "Error parsing %r" % orig)
304: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
305: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = ';'
306: + result = urlparse.parse_qsl(orig)
307: + self.assertEqual(result, expect, "Error parsing %r" % orig)
308: + self.assertEqual(len(w), 0)
309: + with self._qsl_sep_config(';'), catch_warnings(record=True) as w:
310: + result = urlparse.parse_qsl(orig)
311: + self.assertEqual(result, expect, "Error parsing %r" % orig)
312: + self.assertEqual(len(w), 0)
313: +
314: + def test_parse_qs_separator_legacy(self):
315: + for orig, expect in parse_qs_test_cases_legacy:
316: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
317: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
318: + result = urlparse.parse_qs(orig)
319: + self.assertEqual(result, expect, "Error parsing %r" % orig)
320: + self.assertEqual(len(w), 0)
321: + with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
322: + result = urlparse.parse_qs(orig)
323: + self.assertEqual(result, expect, "Error parsing %r" % orig)
324: + self.assertEqual(len(w), 0)
325: +
326: + def test_parse_qsl_separator_legacy(self):
327: + for orig, expect in parse_qsl_test_cases_legacy:
328: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
329: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = 'legacy'
330: + result = urlparse.parse_qsl(orig)
331: + self.assertEqual(result, expect, "Error parsing %r" % orig)
332: + self.assertEqual(len(w), 0)
333: + with self._qsl_sep_config('legacy'), catch_warnings(record=True) as w:
334: + result = urlparse.parse_qsl(orig)
335: + self.assertEqual(result, expect, "Error parsing %r" % orig)
336: + self.assertEqual(len(w), 0)
337: +
338: + def test_parse_qs_separator_bad_value_env_or_config(self):
339: + for bad_sep in '', 'abc', 'safe', '&;', 'SEP':
340: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
341: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = bad_sep
342: + with self.assertRaises(ValueError):
343: + urlparse.parse_qsl('a=1;b=2')
344: + with self._qsl_sep_config('bad_sep'), catch_warnings(record=True) as w:
345: + with self.assertRaises(ValueError):
346: + urlparse.parse_qsl('a=1;b=2')
347: +
348: + def test_parse_qs_separator_bad_value_arg(self):
349: + for bad_sep in True, {}, '':
350: + with self.assertRaises(ValueError):
351: + urlparse.parse_qsl('a=1;b=2', separator=bad_sep)
352: +
353: + def test_parse_qs_separator_num_fields(self):
354: + for qs, sep in (
355: + ('a&b&c', '&'),
356: + ('a;b;c', ';'),
357: + ('a&b;c', 'legacy'),
358: + ):
359: + with EnvironmentVarGuard() as environ, catch_warnings(record=True) as w:
360: + if sep != 'legacy':
361: + with self.assertRaises(ValueError):
362: + urlparse.parse_qsl(qs, separator=sep, max_num_fields=2)
363: + if sep:
364: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = sep
365: + with self.assertRaises(ValueError):
366: + urlparse.parse_qsl(qs, max_num_fields=2)
367: +
368: + def test_parse_qs_separator_priority(self):
369: + # env variable trumps config file
370: + with self._qsl_sep_config('~'), EnvironmentVarGuard() as environ:
371: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = '!'
372: + result = urlparse.parse_qs('a=1!b=2~c=3')
373: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
374: + # argument trumps config file
375: + with self._qsl_sep_config('~'):
376: + result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
377: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
378: + # argument trumps env variable
379: + with EnvironmentVarGuard() as environ:
380: + environ['PYTHON_URLLIB_QS_SEPARATOR'] = '~'
381: + result = urlparse.parse_qs('a=1$b=2~c=3', separator='$')
382: + self.assertEqual(result, {'a': ['1'], 'b': ['2~c=3']})
383: +
384: def test_urlsplit_normalization(self):
385: # Certain characters should never occur in the netloc,
386: # including under normalization.
CVSweb <webmaster@jp.NetBSD.org>