Annotation of pkgsrc/lang/python27/patches/patch-Lib_urlparse.py, revision 1.4
1.4 ! gutterid 1: $NetBSD: patch-Lib_urlparse.py,v 1.3 2022/02/25 22:41:32 gutteridge Exp $
1.1 gutterid 2:
3: Fix CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
4: Via Fedora:
5: https://src.fedoraproject.org/rpms/python2.7/blob/rawhide/f/00359-CVE-2021-23336.patch
6:
1.3 gutterid 7: Fix CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
8: Via Fedora:
9: https://src.fedoraproject.org/rpms/python2.7/raw/40dd05e5d77dbfa81777c9f84b704bc2239bf710/f/00377-CVE-2022-0391.patch
10:
1.4 ! gutterid 11: Fix CVE-2023-24329: Add more sanitizing to respect the "Remove any leading C0 control or space from input" rule
! 12: Via Fedora:
! 13: https://src.fedoraproject.org/rpms/python2.7/c/3f00cdccd59ef2955a7f4b4c42bb59c631cce4c1.patch
! 14:
1.1 gutterid 15: --- Lib/urlparse.py.orig 2020-04-19 21:13:39.000000000 +0000
16: +++ Lib/urlparse.py
1.4 ! gutterid 17: @@ -26,9 +26,14 @@ scenarios for parsing, and for backward
! 18: parsing quirks from older RFCs are retained. The testcases in
! 19: test_urlparse.py provides a good indicator of parsing behavior.
! 20:
! 21: +The WHATWG URL Parser spec should also be considered. We are not compliant with
! 22: +it either due to existing user code API behavior expectations (Hyrum's Law).
! 23: +It serves as a useful guide when making changes.
! 24: +
1.1 gutterid 25: """
26:
27: import re
28: +import os
29:
30: __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
31: "urlsplit", "urlunsplit", "parse_qs", "parse_qsl"]
1.4 ! gutterid 32: @@ -62,6 +67,13 @@ scheme_chars = ('abcdefghijklmnopqrstuvw
1.3 gutterid 33: '0123456789'
34: '+-.')
35:
1.4 ! gutterid 36: +# Leading and trailing C0 control and space to be stripped per WHATWG spec.
! 37: +# == "".join([chr(i) for i in range(0, 0x20 + 1)])
! 38: +_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f '
! 39: +
1.3 gutterid 40: +# Unsafe bytes to be removed per WHATWG spec
41: +_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
42: +
43: MAX_CACHE_SIZE = 20
44: _parse_cache = {}
45:
1.4 ! gutterid 46: @@ -184,12 +196,21 @@ def _checknetloc(netloc):
1.3 gutterid 47: "under NFKC normalization"
48: % netloc)
49:
50: +def _remove_unsafe_bytes_from_url(url):
51: + for b in _UNSAFE_URL_BYTES_TO_REMOVE:
52: + url = url.replace(b, "")
53: + return url
54: +
55: def urlsplit(url, scheme='', allow_fragments=True):
56: """Parse a URL into 5 components:
57: <scheme>://<netloc>/<path>?<query>#<fragment>
58: Return a 5-tuple: (scheme, netloc, path, query, fragment).
59: Note that we don't break the components up in smaller bits
60: (e.g. netloc is a single string) and we don't expand % escapes."""
61: + url = _remove_unsafe_bytes_from_url(url)
62: + scheme = _remove_unsafe_bytes_from_url(scheme)
1.4 ! gutterid 63: + url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE)
! 64: + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE)
1.3 gutterid 65: allow_fragments = bool(allow_fragments)
66: key = url, scheme, allow_fragments, type(url), type(scheme)
67: cached = _parse_cache.get(key, None)
1.4 ! gutterid 68: @@ -382,7 +403,8 @@ def unquote(s):
1.1 gutterid 69: append(item)
70: return ''.join(res)
71:
72: -def parse_qs(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None):
73: +def parse_qs(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None,
74: + separator=None):
75: """Parse a query given as a string argument.
76:
77: Arguments:
1.4 ! gutterid 78: @@ -405,14 +427,23 @@ def parse_qs(qs, keep_blank_values=0, st
1.1 gutterid 79: """
80: dict = {}
81: for name, value in parse_qsl(qs, keep_blank_values, strict_parsing,
82: - max_num_fields):
83: + max_num_fields, separator):
84: if name in dict:
85: dict[name].append(value)
86: else:
87: dict[name] = [value]
88: return dict
89:
90: -def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None):
91: +class _QueryStringSeparatorWarning(RuntimeWarning):
92: + """Warning for using default `separator` in parse_qs or parse_qsl"""
93: +
94: +# The default "separator" for parse_qsl can be specified in a config file.
95: +# It's cached after first read.
1.2 gutterid 96: +_QS_SEPARATOR_CONFIG_FILENAME = '@PKG_SYSCONFDIR@/python/urllib.cfg'
1.1 gutterid 97: +_default_qs_separator = None
98: +
99: +def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None,
100: + separator=None):
101: """Parse a query given as a string argument.
102:
103: Arguments:
1.4 ! gutterid 104: @@ -434,15 +465,72 @@ def parse_qsl(qs, keep_blank_values=0, s
1.1 gutterid 105:
106: Returns a list, as G-d intended.
107: """
108: +
109: + if (not separator or (not isinstance(separator, (str, bytes)))) and separator is not None:
110: + raise ValueError("Separator must be of type string or bytes.")
111: +
112: + # Used when both "&" and ";" act as separators. (Need a non-string value.)
113: + _legacy = object()
114: +
115: + if separator is None:
116: + global _default_qs_separator
117: + separator = _default_qs_separator
118: + envvar_name = 'PYTHON_URLLIB_QS_SEPARATOR'
119: + if separator is None:
120: + # Set default separator from environment variable
121: + separator = os.environ.get(envvar_name)
122: + config_source = 'environment variable'
123: + if separator is None:
124: + # Set default separator from the configuration file
125: + try:
126: + file = open(_QS_SEPARATOR_CONFIG_FILENAME)
127: + except EnvironmentError:
128: + pass
129: + else:
130: + with file:
131: + import ConfigParser
132: + config = ConfigParser.ConfigParser()
133: + config.readfp(file)
134: + separator = config.get('parse_qs', envvar_name)
135: + _default_qs_separator = separator
136: + config_source = _QS_SEPARATOR_CONFIG_FILENAME
137: + if separator is None:
138: + # The default is '&', but warn if not specified explicitly
139: + if ';' in qs:
140: + from warnings import warn
141: + warn("The default separator of urlparse.parse_qsl and "
142: + + "parse_qs was changed to '&' to avoid a web cache "
143: + + "poisoning issue (CVE-2021-23336). "
144: + + "By default, semicolons no longer act as query field "
145: + + "separators. "
146: + + "See https://access.redhat.com/articles/5860431 for "
147: + + "more details.",
148: + _QueryStringSeparatorWarning, stacklevel=2)
149: + separator = '&'
150: + elif separator == 'legacy':
151: + separator = _legacy
152: + elif len(separator) != 1:
153: + raise ValueError(
154: + '{} (from {}) must contain '.format(envvar_name, config_source)
155: + + '1 character, or "legacy". See '
156: + + 'https://access.redhat.com/articles/5860431 for more details.'
157: + )
158: +
159: # If max_num_fields is defined then check that the number of fields
160: # is less than max_num_fields. This prevents a memory exhaustion DOS
161: # attack via post bodies with many fields.
162: if max_num_fields is not None:
163: - num_fields = 1 + qs.count('&') + qs.count(';')
164: + if separator is _legacy:
165: + num_fields = 1 + qs.count('&') + qs.count(';')
166: + else:
167: + num_fields = 1 + qs.count(separator)
168: if max_num_fields < num_fields:
169: raise ValueError('Max number of fields exceeded')
170:
171: - pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
172: + if separator is _legacy:
173: + pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
174: + else:
175: + pairs = [s1 for s1 in qs.split(separator)]
176: r = []
177: for name_value in pairs:
178: if not name_value and not strict_parsing:
CVSweb <webmaster@jp.NetBSD.org>